mm: remove __ARCH_HAS_5LEVEL_HACK
Subsystems (14)
Mike Rapoport
reviewerv3 cover mm: remove __ARCH_HAS_5LEVEL_HACK
From: Mike Rapoport <[email protected]>
Hi,
These patches convert several architectures to use page table folding and remove __ARCH_HAS_5LEVEL_HACK along with include/asm-generic/5level-fixup.h and include/asm-generic/pgtable-nop4d-hack.h. With that we'll have a single and consistent way of dealing with page table folding instead of a mix of three existing options.
The changes are mostly about mechanical replacement of pgd accessors with p4d ones and the addition of higher levels to page table traversals.
v3:
* add Christophe's patch that removes ppc32 get_pteptr()
* reduce amount of upper layer walks in powerpc
v2:
* collect per-arch patches into a single set
* include Geert's update of 'sh' printing messages
* rebase on v5.6-rc1+
Christophe Leroy (1):
powerpc/32: drop get_pteptr()
Geert Uytterhoeven (1):
sh: fault: Modernize printing of kernel messages
Mike Rapoport (12):
arm/arm64: add support for folded p4d page tables
h8300: remove usage of __ARCH_USE_5LEVEL_HACK
hexagon: remove __ARCH_USE_5LEVEL_HACK
ia64: add support for folded p4d page tables
nios2: add support for folded p4d page tables
openrisc: add support for folded p4d page tables
powerpc: add support for folded p4d page tables
sh: drop __pXd_offset() macros that duplicate pXd_index() ones
sh: add support for folded p4d page tables
unicore32: remove __ARCH_USE_5LEVEL_HACK
asm-generic: remove pgtable-nop4d-hack.h
mm: remove __ARCH_HAS_5LEVEL_HACK and
include/asm-generic/5level-fixup.h
arch/arm/include/asm/kvm_mmu.h | 5 +- arch/arm/include/asm/pgtable.h | 1 - arch/arm/include/asm/stage2_pgtable.h | 15 +- arch/arm/lib/uaccess_with_memcpy.c | 9 +- arch/arm/mach-sa1100/assabet.c | 2 +- arch/arm/mm/dump.c | 29 ++- arch/arm/mm/fault-armv.c | 7 +- arch/arm/mm/fault.c | 28 ++- arch/arm/mm/idmap.c | 3 +- arch/arm/mm/init.c | 2 +- arch/arm/mm/ioremap.c | 12 +- arch/arm/mm/mm.h | 2 +- arch/arm/mm/mmu.c | 35 ++- arch/arm/mm/pgd.c | 40 +++- arch/arm64/include/asm/kvm_mmu.h | 10 +- arch/arm64/include/asm/pgalloc.h | 10 +- arch/arm64/include/asm/pgtable-types.h | 5 +- arch/arm64/include/asm/pgtable.h | 37 ++-- arch/arm64/include/asm/stage2_pgtable.h | 48 +++- arch/arm64/kernel/hibernate.c | 44 +++- arch/arm64/mm/fault.c | 9 +- arch/arm64/mm/hugetlbpage.c | 15 +- arch/arm64/mm/kasan_init.c | 26 ++- arch/arm64/mm/mmu.c | 52 +++-- arch/arm64/mm/pageattr.c | 7 +- arch/h8300/include/asm/pgtable.h | 1 - arch/hexagon/include/asm/fixmap.h | 4 +- arch/hexagon/include/asm/pgtable.h | 1 - arch/ia64/include/asm/pgalloc.h | 4 +- arch/ia64/include/asm/pgtable.h | 17 +- arch/ia64/mm/fault.c | 7 +- arch/ia64/mm/hugetlbpage.c | 18 +- arch/ia64/mm/init.c | 28 ++- arch/nios2/include/asm/pgtable.h | 3 +- arch/nios2/mm/fault.c | 9 +- arch/nios2/mm/ioremap.c | 6 +- arch/openrisc/include/asm/pgtable.h | 1 - arch/openrisc/mm/fault.c | 10 +- arch/openrisc/mm/init.c | 4 +- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 60 ++--- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +-- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 6 +- arch/powerpc/kvm/book3s_64_mmu_radix.c | 30 ++- arch/powerpc/lib/code-patching.c | 7 +- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 26 ++- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 ++- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +- arch/powerpc/mm/pgtable.c | 30 ++- arch/powerpc/mm/pgtable_32.c | 45 +--- arch/powerpc/mm/pgtable_64.c | 10 +- arch/powerpc/mm/ptdump/hashpagetable.c | 20 +- arch/powerpc/mm/ptdump/ptdump.c | 14 +- arch/powerpc/xmon/xmon.c | 18 +- arch/sh/include/asm/pgtable-2level.h | 1 - arch/sh/include/asm/pgtable-3level.h | 1 - arch/sh/include/asm/pgtable_32.h | 5 +- arch/sh/include/asm/pgtable_64.h | 5 +- arch/sh/kernel/io_trapped.c | 7 +- arch/sh/mm/cache-sh4.c | 4 +- arch/sh/mm/cache-sh5.c | 7 +- arch/sh/mm/fault.c | 65 ++++-- arch/sh/mm/hugetlbpage.c | 28 ++- arch/sh/mm/init.c | 15 +- arch/sh/mm/kmap.c | 2 +- arch/sh/mm/tlbex_32.c | 6 +- arch/sh/mm/tlbex_64.c | 7 +- arch/unicore32/include/asm/pgtable.h | 1 - arch/unicore32/kernel/hibernate.c | 4 +- include/asm-generic/5level-fixup.h | 58 ----- include/asm-generic/pgtable-nop4d-hack.h | 64 ------ include/asm-generic/pgtable-nopud.h | 4 - include/linux/mm.h | 6 - mm/kasan/init.c | 11 - mm/memory.c | 8 - virt/kvm/arm/mmu.c | 209 +++++++++++++++--- 89 files changed, 917 insertions(+), 564 deletions(-) delete mode 100644 include/asm-generic/5level-fixup.h delete mode 100644 include/asm-generic/pgtable-nop4d-hack.h
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 1/14 arm/arm64: add support for folded p4d page tables
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate, replace 5level-fixup.h with pgtable-nop4d.h and remove __ARCH_USE_5LEVEL_HACK.
Since arm and arm64 share kvm memory management bits, make the conversion for both variants at once to avoid breaking the builds in the middle.
--- arch/arm/include/asm/kvm_mmu.h | 5 +- arch/arm/include/asm/pgtable.h | 1 - arch/arm/include/asm/stage2_pgtable.h | 15 +- arch/arm/lib/uaccess_with_memcpy.c | 9 +- arch/arm/mach-sa1100/assabet.c | 2 +- arch/arm/mm/dump.c | 29 +++- arch/arm/mm/fault-armv.c | 7 +- arch/arm/mm/fault.c | 28 +++- arch/arm/mm/idmap.c | 3 +- arch/arm/mm/init.c | 2 +- arch/arm/mm/ioremap.c | 12 +- arch/arm/mm/mm.h | 2 +- arch/arm/mm/mmu.c | 35 +++- arch/arm/mm/pgd.c | 40 ++++- arch/arm64/include/asm/kvm_mmu.h | 10 +- arch/arm64/include/asm/pgalloc.h | 10 +- arch/arm64/include/asm/pgtable-types.h | 5 +- arch/arm64/include/asm/pgtable.h | 37 +++-- arch/arm64/include/asm/stage2_pgtable.h | 48 ++++-- arch/arm64/kernel/hibernate.c | 44 ++++- arch/arm64/mm/fault.c | 9 +- arch/arm64/mm/hugetlbpage.c | 15 +- arch/arm64/mm/kasan_init.c | 26 ++- arch/arm64/mm/mmu.c | 52 ++++-- arch/arm64/mm/pageattr.c | 7 +- virt/kvm/arm/mmu.c | 209 ++++++++++++++++++++---- 26 files changed, 522 insertions(+), 140 deletions(-)
arch/arm/include/asm/kvm_mmu.h +4 -1
@@ -68,7 +68,8 @@ void kvm_clear_hyp_idmap(void);#define kvm_mk_pmd (ptep) __pmd(__pa(ptep) | PMD_TYPE_TABLE)#define kvm_mk_pud (pmdp) __pud(__pa(pmdp) | PMD_TYPE_TABLE) -#define kvm_mk_pgd (pudp) ({ BUILD_BUG(); 0; }) +#define kvm_mk_p4d (pudp) ({ BUILD_BUG(); __p4d(0); }) +#define kvm_mk_pgd (p4dp) ({ BUILD_BUG(); 0; })#define kvm_pfn_pte (pfn, prot) pfn_pte(pfn, prot)#define kvm_pfn_pmd (pfn, prot) pfn_pmd(pfn, prot)
@@ -194,10 +195,12 @@ static inline bool kvm_page_empty(void *ptr)#define kvm_pte_table_empty (kvm, ptep) kvm_page_empty(ptep)#define kvm_pmd_table_empty (kvm, pmdp) kvm_page_empty(pmdp)#define kvm_pud_table_empty (kvm, pudp) false +#define kvm_p4d_table_empty (kvm, p4dp) false#define hyp_pte_table_empty (ptep) kvm_page_empty(ptep)#define hyp_pmd_table_empty (pmdp) kvm_page_empty(pmdp)#define hyp_pud_table_empty (pudp) false +#define hyp_p4d_table_empty (p4dp) falsestruct kvm ;
arch/arm/include/asm/pgtable.h +0 -1
@@ -17,7 +17,6 @@#else -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopud.h> #include <asm/memory.h> #include <asm/pgtable-hwdef.h>
arch/arm/include/asm/stage2_pgtable.h +13 -2
@@ -19,8 +19,17 @@#define stage2_pgd_none (kvm, pgd) pgd_none(pgd)#define stage2_pgd_clear (kvm, pgd) pgd_clear(pgd)#define stage2_pgd_present (kvm, pgd) pgd_present(pgd) -#define stage2_pgd_populate (kvm, pgd, pud) pgd_populate(NULL, pgd, pud) -#define stage2_pud_offset (kvm, pgd, address) pud_offset(pgd, address) +#define stage2_pgd_populate (kvm, pgd, p4d) pgd_populate(NULL, pgd, p4d) + +#define stage2_p4d_offset (kvm, pgd, address) p4d_offset(pgd, address) +#define stage2_p4d_free (kvm, p4d) do { } while (0) + +#define stage2_p4d_none (kvm, p4d) p4d_none(p4d) +#define stage2_p4d_clear (kvm, p4d) p4d_clear(p4d) +#define stage2_p4d_present (kvm, p4d) p4d_present(p4d) +#define stage2_p4d_populate (kvm, p4d, pud) p4d_populate(NULL, p4d, pud) + +#define stage2_pud_offset (kvm, p4d, address) pud_offset(p4d, address)#define stage2_pud_free (kvm, pud) do { } while (0)#define stage2_pud_none (kvm, pud) pud_none(pud)
@@ -41,6 +50,7 @@ stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)return (boundary -1 < end -1 ) ? boundary : end; } +#define stage2_p4d_addr_end (kvm, addr, end) (end)#define stage2_pud_addr_end (kvm, addr, end) (end) static inline phys_addr_t
@@ -56,6 +66,7 @@ stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)#define stage2_pte_table_empty (kvm, ptep) kvm_page_empty(ptep)#define stage2_pmd_table_empty (kvm, pmdp) kvm_page_empty(pmdp)#define stage2_pud_table_empty (kvm, pudp) false +#define stage2_p4d_table_empty (kvm, p4dp) falsestatic inline bool kvm_stage2_has_pud (struct kvm *kvm) {
arch/arm/lib/uaccess_with_memcpy.c +7 -2
@@ -24,6 +24,7 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) {unsigned long addr = (unsigned long )_addr;pgd_t *pgd; +p4d_t *p4d;pmd_t *pmd;pte_t *pte;pud_t *pud;
@@ -33,7 +34,11 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)if (unlikely (pgd_none (*pgd) ||pgd_bad (*pgd)))return 0 ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (unlikely (p4d_none (*p4d) ||p4d_bad (*p4d))) +return 0 ; + + pud =pud_offset (p4d, addr);if (unlikely (pud_none (*pud) ||pud_bad (*pud)))return 0 ;
@@ -154,7 +159,7 @@ arm_copy_to_user(void __user *to, const void *from, unsigned long n) }return n; } - +static unsigned long noinline __clear_user_memset (void __user *addr,unsigned long n) {
arch/arm/mach-sa1100/assabet.c +1 -1
@@ -633,7 +633,7 @@ static void __init map_sa1100_gpio_regs( void )int prot =PMD_TYPE_SECT |PMD_SECT_AP_WRITE |PMD_DOMAIN (DOMAIN_IO );pmd_t *pmd; - pmd =pmd_offset (pud_offset (pgd_offset_k (virt), virt), virt); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (virt), virt), virt), virt); *pmd =__pmd (phys | prot);flush_pmd_entry (pmd); }
arch/arm/mm/dump.c +23 -6
@@ -207,6 +207,7 @@ struct pg_level {static struct pg_level pg_level[] = { { }, {/* pgd */ + }, {/* p4d */ }, {/* pud */ }, {/* pmd */ .bits = section_bits,
@@ -308,7 +309,7 @@ static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start,for (i =0 ; i <PTRS_PER_PTE ; i++, pte++) { addr = start + i *PAGE_SIZE ; -note_page (st, addr,4 ,pte_val (*pte), domain); +note_page (st, addr,5 ,pte_val (*pte), domain); } }
@@ -350,14 +351,14 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) addr +=SECTION_SIZE ; pmd++; domain =get_domain_name (pmd); -note_page (st, addr,3 ,pmd_val (*pmd), domain); +note_page (st, addr,4 ,pmd_val (*pmd), domain); } } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned i;
@@ -366,7 +367,23 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)if (!pud_none (*pud)) {walk_pmd (st, pud, addr); }else { -note_page (st, addr,2 ,pud_val (*pud),NULL ); +note_page (st, addr,3 ,pud_val (*pud),NULL ); + } + } +} + +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) { +walk_pud (st, p4d, addr); + }else { +note_page (st, addr,2 ,p4d_val (*p4d),NULL ); } } }
@@ -381,7 +398,7 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm,for (i =0 ; i <PTRS_PER_PGD ; i++, pgd++) { addr = start + i *PGDIR_SIZE ;if (!pgd_none (*pgd)) { -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); }else {note_page (st, addr,1 ,pgd_val (*pgd),NULL ); }
arch/arm/mm/fault-armv.c +6 -1
@@ -91,6 +91,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, {spinlock_t *ptl;pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -100,7 +101,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,if (pgd_none_or_clear_bad (pgd))return 0 ; - pud =pud_offset (pgd, address); + p4d =p4d_offset (pgd, address); +if (p4d_none_or_clear_bad (p4d)) +return 0 ; + + pud =pud_offset (p4d, address);if (pud_none_or_clear_bad (pud))return 0 ;
arch/arm/mm/fault.c +25 -3
@@ -43,6 +43,7 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)printk ("%s[%08lx] *pgd=%08llx" , lvl, addr, (long long )pgd_val (*pgd)); do { + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte;
@@ -55,7 +56,19 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)break ; } - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (PTRS_PER_P4D !=1 ) +pr_cont (", *p4d=%08llx" , (long long )p4d_val (*p4d)); + +if (p4d_none (*p4d)) +break ; + +if (p4d_bad (*p4d)) { +pr_cont ("(bad)" ); +break ; + } + + pud =pud_offset (p4d, addr);if (PTRS_PER_PUD !=1 )pr_cont (", *pud=%08llx" , (long long )pud_val (*pud));
@@ -408,6 +421,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr, {unsigned int index;pgd_t *pgd, *pgd_k; +p4d_t *p4d, *p4d_k;pud_t *pud, *pud_k;pmd_t *pmd, *pmd_k;
@@ -427,8 +441,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr,if (!pgd_present (*pgd))set_pgd (pgd, *pgd_k); - pud =pud_offset (pgd, addr); - pud_k =pud_offset (pgd_k, addr); + p4d =p4d_offset (pgd, addr); + p4d_k =p4d_offset (pgd_k, addr); + +if (p4d_none (*p4d_k)) + goto bad_area; +if (!p4d_present (*p4d)) +set_p4d (p4d, *p4d_k); + + pud =pud_offset (p4d, addr); + pud_k =pud_offset (p4d_k, addr);if (pud_none (*pud_k)) goto bad_area;
arch/arm/mm/idmap.c +2 -1
@@ -68,7 +68,8 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,static void idmap_add_pud (pgd_t *pgd,unsigned long addr,unsigned long end,unsigned long prot) { -pud_t *pud =pud_offset (pgd, addr); +p4d_t *p4d =p4d_offset (pgd, addr); +pud_t *pud =pud_offset (p4d, addr);unsigned long next; do {
arch/arm/mm/init.c +1 -1
@@ -571,7 +571,7 @@ static inline void section_update(unsigned long addr, pmdval_t mask, {pmd_t *pmd; - pmd =pmd_offset (pud_offset (pgd_offset (mm, addr), addr), addr); + pmd =pmd_off_k (addr);#ifdef CONFIG_ARM_LPAE pmd[0 ] =__pmd ((pmd_val (pmd[0 ]) & mask) | prot);
arch/arm/mm/ioremap.c +9 -3
@@ -142,12 +142,14 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) {unsigned long addr = virt, end = virt + (size & ~(SZ_1M -1 ));pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmdp;flush_cache_vunmap (addr, end); pgd =pgd_offset_k (addr); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); + pud =pud_offset (p4d, addr); pmdp =pmd_offset (pud, addr); do { pmd_t pmd = *pmdp;
@@ -190,6 +192,7 @@ remap_area_sections(unsigned long virt, unsigned long pfn, {unsigned long addr = virt, end = virt + size;pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -200,7 +203,8 @@ remap_area_sections(unsigned long virt, unsigned long pfn,unmap_area_sections (virt, size); pgd =pgd_offset_k (addr); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); + pud =pud_offset (p4d, addr); pmd =pmd_offset (pud, addr); do { pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
@@ -222,6 +226,7 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, {unsigned long addr = virt, end = virt + size;pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -232,7 +237,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,unmap_area_sections (virt, size); pgd =pgd_offset_k (virt); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); + pud =pud_offset (p4d, addr); pmd =pmd_offset (pud, addr); do { unsigned long super_pmd_val, i;
arch/arm/mm/mm.h +1 -1
@@ -38,7 +38,7 @@ static inline pte_t get_top_pte(unsigned long va)static inline pmd_t *pmd_off_k (unsigned long virt) { -return pmd_offset (pud_offset (pgd_offset_k (virt), virt), virt); +return pmd_offset (pud_offset (p4d_offset (pgd_offset_k (virt), virt), virt), virt); }struct mem_type {
arch/arm/mm/mmu.c +29 -6
@@ -375,7 +375,8 @@ static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr)static inline pmd_t *__init fixmap_pmd (unsigned long addr) {pgd_t *pgd =pgd_offset_k (addr); -pud_t *pud =pud_offset (pgd, addr); +p4d_t *p4d =p4d_offset (pgd, addr); +pud_t *pud =pud_offset (p4d, addr);pmd_t *pmd =pmd_offset (pud, addr);return pmd;
@@ -827,12 +828,12 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, }while (pmd++, addr = next, addr != end); } -static void __init alloc_init_pud (pgd_t *pgd,unsigned long addr, +static void __init alloc_init_pud (p4d_t *p4d,unsigned long addr,unsigned long end,phys_addr_t phys,const struct mem_type *type,void *(*alloc)(unsigned long sz),bool ng) { -pud_t *pud =pud_offset (pgd, addr); +pud_t *pud =pud_offset (p4d, addr);unsigned long next; do {
@@ -842,6 +843,21 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, }while (pud++, addr = next, addr != end); } +static void __init alloc_init_p4d (pgd_t *pgd,unsigned long addr, +unsigned long end,phys_addr_t phys, +const struct mem_type *type, +void *(*alloc)(unsigned long sz),bool ng) +{ +p4d_t *p4d =p4d_offset (pgd, addr); +unsigned long next; + +do { + next =p4d_addr_end (addr, end); +alloc_init_pud (p4d, addr, next, phys, type, alloc, ng); + phys += next - addr; + }while (p4d++, addr = next, addr != end); +} +#ifndef CONFIG_ARM_LPAE static void __init create_36bit_mapping(struct mm_struct *mm, struct map_desc *md,
@@ -889,7 +905,8 @@ static void __init create_36bit_mapping(struct mm_struct *mm, pgd =pgd_offset (mm, addr); end = addr + length; do { - pud_t *pud = pud_offset(pgd, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); pmd_t *pmd = pmd_offset(pud, addr); int i;
@@ -940,7 +957,7 @@ static void __init __create_mapping(struct mm_struct *mm, struct map_desc *md, do { unsigned long next = pgd_addr_end(addr, end); - alloc_init_pud(pgd, addr, next, phys, type, alloc, ng); + alloc_init_p4d(pgd, addr, next, phys, type, alloc, ng); phys += next - addr; addr = next;
@@ -976,7 +993,13 @@ void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md, bool ng) {#ifdef CONFIG_ARM_LPAE -pud_t *pud =pud_alloc (mm,pgd_offset (mm, md->virtual), md->virtual); +p4d_t *p4d; +pud_t *pud; + + p4d =p4d_alloc (mm,pgd_offset (mm, md->virtual), md->virtual); +if (!WARN_ON (!p4d)) +return ; + pud =pud_alloc (mm, p4d, md->virtual);if (WARN_ON (!pud))return ;pmd_alloc (mm, pud,0 );
arch/arm/mm/pgd.c +33 -7
@@ -30,6 +30,7 @@pgd_t *pgd_alloc (struct mm_struct *mm) {pgd_t *new_pgd, *init_pgd; +p4d_t *new_p4d, *init_p4d;pud_t *new_pud, *init_pud;pmd_t *new_pmd, *init_pmd;pte_t *new_pte, *init_pte;
@@ -53,8 +54,12 @@ pgd_t *pgd_alloc(struct mm_struct *mm)/* * Allocate PMD table for modules and pkmap mappings. */ - new_pud =pud_alloc (mm, new_pgd +pgd_index (MODULES_VADDR ), + new_p4d =p4d_alloc (mm, new_pgd +pgd_index (MODULES_VADDR ),MODULES_VADDR ); +if (!new_p4d) + goto no_p4d; + + new_pud =pud_alloc (mm, new_p4d,MODULES_VADDR );if (!new_pud) goto no_pud;
@@ -69,7 +74,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm)* contains the machine vectors. The vectors are always high * with LPAE. */ - new_pud =pud_alloc (mm, new_pgd,0 ); + new_p4d =p4d_alloc (mm, new_pgd,0 ); +if (!new_p4d) + goto no_p4d; + + new_pud =pud_alloc (mm, new_p4d,0 );if (!new_pud) goto no_pud;
@@ -91,7 +100,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)pmd_val (*new_pmd) |=PMD_DOMAIN (DOMAIN_VECTORS );#endif - init_pud =pud_offset (init_pgd,0 ); + init_p4d =p4d_offset (init_pgd,0 ); + init_pud =pud_offset (init_p4d,0 ); init_pmd =pmd_offset (init_pud,0 ); init_pte =pte_offset_map (init_pmd,0 );set_pte_ext (new_pte +0 , init_pte[0 ],0 );
@@ -108,6 +118,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) no_pmd:pud_free (mm, new_pud); no_pud: +p4d_free (mm, new_p4d); +no_p4d:__pgd_free (new_pgd); no_pgd:return NULL ;
@@ -116,6 +128,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)void pgd_free (struct mm_struct *mm,pgd_t *pgd_base) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pgtable_t pte;
@@ -127,7 +140,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)if (pgd_none_or_clear_bad (pgd)) goto no_pgd; - pud =pud_offset (pgd,0 ); + p4d =p4d_offset (pgd,0 ); +if (p4d_none_or_clear_bad (p4d)) + goto no_p4d; + + pud =pud_offset (p4d,0 );if (pud_none_or_clear_bad (pud)) goto no_pud;
@@ -144,8 +161,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)pmd_free (mm, pmd);mm_dec_nr_pmds (mm); no_pud: -pgd_clear (pgd); +p4d_clear (p4d);pud_free (mm, pud); +no_p4d: +pgd_clear (pgd); +p4d_free (mm, p4d); no_pgd: #ifdef CONFIG_ARM_LPAE/*
@@ -156,15 +176,21 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)continue ;if (pgd_val (*pgd) &L_PGD_SWAPPER )continue ; - pud =pud_offset (pgd,0 ); + p4d =p4d_offset (pgd,0 ); +if (p4d_none_or_clear_bad (p4d)) +continue ; + pud =pud_offset (p4d,0 );if (pud_none_or_clear_bad (pud))continue ; pmd =pmd_offset (pud,0 );pud_clear (pud);pmd_free (mm, pmd);mm_dec_nr_pmds (mm); -pgd_clear (pgd); +p4d_clear (p4d);pud_free (mm, pud); +mm_dec_nr_puds (mm); +pgd_clear (pgd); + p4d_free(mm, p4d); }#endif __pgd_free (pgd_base);
arch/arm64/include/asm/kvm_mmu.h +8 -2
@@ -172,8 +172,8 @@ void kvm_clear_hyp_idmap(void); __pmd(__phys_to_pmd_val(__pa(ptep)) | PMD_TYPE_TABLE)#define kvm_mk_pud (pmdp) \ __pud(__phys_to_pud_val(__pa(pmdp)) | PMD_TYPE_TABLE) -#define kvm_mk_pgd (pudp) \ - __pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE) +#define kvm_mk_p4d (pmdp) \ + __p4d(__phys_to_p4d_val(__pa(pmdp)) | PUD_TYPE_TABLE)#define kvm_set_pud (pudp, pud) set_pud(pudp, pud)
@@ -299,6 +299,12 @@ static inline bool kvm_s2pud_young(pud_t pud)#define hyp_pud_table_empty (pudp) kvm_page_empty(pudp)#endif +#ifdef __PAGETABLE_P4D_FOLDED +#define hyp_p4d_table_empty (p4dp) (0) +#else +#define hyp_p4d_table_empty (p4dp) kvm_page_empty(p4dp) +#endif +struct kvm ;#define kvm_flush_dcache_to_poc (a,l) __flush_dcache_area((a), (l))
arch/arm64/include/asm/pgalloc.h +5 -5
@@ -73,17 +73,17 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pudp) free_page((unsigned long)pudp); } -static inline void __pgd_populate (pgd_t *pgdp,phys_addr_t pudp,pgdval_t prot) +static inline void __p4d_populate (p4d_t *p4dp,phys_addr_t pudp,p4dval_t prot) { -set_pgd (pgdp,__pgd (__phys_to_pgd_val (pudp) | prot)); +set_p4d (p4dp,__p4d (__phys_to_p4d_val (pudp) | prot)); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgdp,pud_t *pudp) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *p4dp,pud_t *pudp) { -__pgd_populate (pgdp,__pa (pudp),PUD_TYPE_TABLE ); +__p4d_populate (p4dp,__pa (pudp),PUD_TYPE_TABLE ); }#else -static inline void __pgd_populate (pgd_t *pgdp,phys_addr_t pudp,pgdval_t prot) +static inline void __p4d_populate (p4d_t *p4dp,phys_addr_t pudp,p4dval_t prot) {BUILD_BUG (); }
arch/arm64/include/asm/pgtable-types.h +2 -3
@@ -14,6 +14,7 @@typedef u64 pteval_t ;typedef u64 pmdval_t ;typedef u64 pudval_t ; +typedef u64 p4dval_t ;typedef u64 pgdval_t ;/*
@@ -44,13 +45,11 @@ typedef struct { pteval_t pgprot; } pgprot_t;#define __pgprot (x) ((pgprot_t) { (x) } ) #if CONFIG_PGTABLE_LEVELS = 2 -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #elif CONFIG_PGTABLE_LEVELS = 3 -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopud.h> #elif CONFIG_PGTABLE_LEVELS = 4 -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> #endif #endif /* __ASM_PGTABLE_TYPES_H */
arch/arm64/include/asm/pgtable.h +23 -14
@@ -298,6 +298,11 @@ static inline pte_t pgd_pte(pgd_t pgd)return __pte (pgd_val (pgd)); } +static inline pte_t p4d_pte (p4d_t p4d) +{ +return __pte (p4d_val (p4d)); +} +static inline pte_t pud_pte (pud_t pud) {return __pte (pud_val (pud));
@@ -401,6 +406,9 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)#define set_pmd_at (mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd)) +#define __p4d_to_phys (p4d) __pte_to_phys(p4d_pte(p4d)) +#define __phys_to_p4d_val (phys) __phys_to_pte_val(phys) +#define __pgd_to_phys (pgd) __pte_to_phys(pgd_pte(pgd))#define __phys_to_pgd_val (phys) __phys_to_pte_val(phys)
@@ -588,49 +596,50 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)#define pud_ERROR (pud) __pud_error(__FILE__, __LINE__, pud_val(pud)) -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (!(pgd_val(pgd) & 2)) -#define pgd_present (pgd) (pgd_val(pgd)) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (!(p4d_val(p4d) & 2)) +#define p4d_present (p4d) (p4d_val(p4d)) -static inline void set_pgd (pgd_t *pgdp,pgd_t pgd) +static inline void set_p4d (p4d_t *p4dp,p4d_t p4d) { -if (in_swapper_pgdir (pgdp)) { -set_swapper_pgd (pgdp, pgd); +if (in_swapper_pgdir (p4dp)) { +set_swapper_pgd ((pgd_t *)p4dp,__pgd (p4d_val (p4d)));return ; } -WRITE_ONCE (*pgdp, pgd); +WRITE_ONCE (*p4dp, p4d);dsb (ishst);isb (); } -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { -set_pgd (pgdp,__pgd (0 )); +set_p4d (p4dp,__p4d (0 )); } -static inline phys_addr_t pgd_page_paddr (pgd_t pgd) +static inline phys_addr_t p4d_page_paddr (p4d_t p4d) { -return __pgd_to_phys (pgd); +return __p4d_to_phys (p4d); }/* Find an entry in the frst-level page table. */ #define pud_index (addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) -#define pud_offset_phys (dir, addr) (pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t)) +#define pud_offset_phys (dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))#define pud_offset (dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr))))#define pud_set_fixmap (addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) -#define pud_set_fixmap_offset (pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr)) +#define pud_set_fixmap_offset (p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr))#define pud_clear_fixmap () clear_fixmap(FIX_PUD) -#define pgd_page (pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd))) +#define p4d_page (p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))/* use ONLY for statically allocated translation tables */ #define pud_offset_kimg (dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))#else +#define p4d_page_paddr (p4d) ({ BUILD_BUG(); 0;})#define pgd_page_paddr (pgd) ({ BUILD_BUG(); 0;})/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
arch/arm64/include/asm/stage2_pgtable.h +37 -11
78 lines not on this page · raw message
arch/arm64/kernel/hibernate.c +37 -7
85 lines not on this page · raw message
arch/arm64/mm/fault.c +8 -1
@@ -145,6 +145,7 @@ static void show_pte(unsigned long addr)pr_alert ("[%016lx] pgd=%016llx" , addr,pgd_val (pgd)); do { + p4d_t *p4dp, p4d; pud_t *pudp, pud; pmd_t *pmdp, pmd; pte_t *ptep, pte;
@@ -152,7 +153,13 @@ static void show_pte(unsigned long addr)if (pgd_none (pgd) ||pgd_bad (pgd))break ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + p4d =READ_ONCE (*p4dp); +pr_cont (", p4d=%016llx" ,p4d_val (p4d)); +if (p4d_none (p4d) ||p4d_bad (p4d)) +break ; + + pudp =pud_offset (p4dp, addr); pud =READ_ONCE (*pudp);pr_cont (", pud=%016llx" ,pud_val (pud));if (pud_none (pud) ||pud_bad (pud))
arch/arm64/mm/hugetlbpage.c +12 -3
@@ -67,11 +67,13 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, pte_t *ptep, size_t *pgsize) {pgd_t *pgdp =pgd_offset (mm, addr); +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp; *pgsize =PAGE_SIZE ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + pudp =pud_offset (p4dp, addr); pmdp =pmd_offset (pudp, addr);if ((pte_t *)pmdp = ptep) { *pgsize =PMD_SIZE ;
@@ -217,12 +219,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep =NULL ; pgdp =pgd_offset (mm, addr); - pudp =pud_alloc (mm, pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + pudp =pud_alloc (mm, p4dp, addr);if (!pudp)return NULL ;
@@ -259,6 +263,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp, pud;pmd_t *pmdp, pmd;
@@ -266,7 +271,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm,if (!pgd_present (READ_ONCE (*pgdp)))return NULL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (!p4d_present (READ_ONCE (*p4dp))) +return NULL ; + + pudp =pud_offset (p4dp, addr); pud =READ_ONCE (*pudp);if (sz !=PUD_SIZE &&pud_none (pud))return NULL ;
arch/arm64/mm/kasan_init.c +19 -7
@@ -84,17 +84,17 @@ static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,return early ?pmd_offset_kimg (pudp, addr) :pmd_offset (pudp, addr); } -static pud_t *__init kasan_pud_offset (pgd_t *pgdp,unsigned long addr,int node, +static pud_t *__init kasan_pud_offset (p4d_t *p4dp,unsigned long addr,int node,bool early) { -if (pgd_none (READ_ONCE (*pgdp))) { +if (p4d_none (READ_ONCE (*p4dp))) {phys_addr_t pud_phys = early ?__pa_symbol (kasan_early_shadow_pud) :kasan_alloc_zeroed_page (node); -__pgd_populate (pgdp, pud_phys,PMD_TYPE_TABLE ); +__p4d_populate (p4dp, pud_phys,PMD_TYPE_TABLE ); } -return early ?pud_offset_kimg (pgdp, addr) :pud_offset (pgdp, addr); +return early ?pud_offset_kimg (p4dp, addr) :pud_offset (p4dp, addr); } static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
@@ -126,11 +126,11 @@ static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, }while (pmdp++, addr = next, addr != end &&pmd_none (READ_ONCE (*pmdp))); } -static void __init kasan_pud_populate (pgd_t *pgdp,unsigned long addr, +static void __init kasan_pud_populate (p4d_t *p4dp,unsigned long addr,unsigned long end,int node,bool early) {unsigned long next; -pud_t *pudp =kasan_pud_offset (pgdp, addr, node, early); +pud_t *pudp =kasan_pud_offset (p4dp, addr, node, early); do { next = pud_addr_end(addr, end);
@@ -138,6 +138,18 @@ static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr, }while (pudp++, addr = next, addr != end &&pud_none (READ_ONCE (*pudp))); } +static void __init kasan_p4d_populate (pgd_t *pgdp,unsigned long addr, +unsigned long end,int node,bool early) +{ +unsigned long next; +p4d_t *p4dp =p4d_offset (pgdp, addr); + +do { + next =p4d_addr_end (addr, end); +kasan_pud_populate (p4dp, addr, next, node, early); + }while (p4dp++, addr = next, addr != end); +} +static void __init kasan_pgd_populate (unsigned long addr,unsigned long end,int node,bool early) {
@@ -147,7 +159,7 @@ static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, pgdp =pgd_offset_k (addr);do { next =pgd_addr_end (addr, end); -kasan_pud_populate (pgdp, addr, next, node, early); +kasan_p4d_populate (pgdp, addr, next, node, early); }while (pgdp++, addr = next, addr != end); }
arch/arm64/mm/mmu.c +34 -18
127 lines not on this page · raw message
arch/arm64/mm/pageattr.c +6 -1
@@ -198,6 +198,7 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)bool kernel_page_present (struct page *page) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp, pud;pmd_t *pmdp, pmd;pte_t *ptep;
@@ -210,7 +211,11 @@ bool kernel_page_present(struct page *page)if (pgd_none (READ_ONCE (*pgdp)))return false; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (p4d_none (READ_ONCE (*p4dp))) +return false; + + pudp =pud_offset (p4dp, addr); pud =READ_ONCE (*pudp);if (pud_none (pud))return false;
virt/kvm/arm/mmu.c +177 -32
388 lines not on this page · raw message
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 2/14 h8300: remove usage of __ARCH_USE_5LEVEL_HACK
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
h8300 is a nommu architecture and does not require fixup for upper layers of the page tables because it is already handled by the generic nommu implementation.
Remove definition of __ARCH_USE_5LEVEL_HACK inarch/h8300/include/asm/pgtable.h
--- arch/h8300/include/asm/pgtable.h | 1 - 1 file changed, 1 deletion(-)
arch/h8300/include/asm/pgtable.h +0 -1
@@ -1,7 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _H8300_PGTABLE_H#define _H8300_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopud.h> #include <asm-generic/pgtable.h> extern void paging_init (void );
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 3/14 hexagon: remove __ARCH_USE_5LEVEL_HACK
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
The hexagon architecture has 2 level page tables and as such most of the page table folding is already implemented in asm-generic/pgtable-nopmd.h.
Fixup the only place in arch/hexagon to unfold the p4d level and remove __ARCH_USE_5LEVEL_HACK.
--- arch/hexagon/include/asm/fixmap.h | 4 ++-- arch/hexagon/include/asm/pgtable.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-)
arch/hexagon/include/asm/fixmap.h +2 -2
@@ -16,7 +16,7 @@#include <asm-generic/fixmap.h> #define kmap_get_fixmap_pte (vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), \ - (vaddr)), (vaddr)), (vaddr)) + pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), \ + (vaddr)), (vaddr)), (vaddr)), (vaddr))#endif
arch/hexagon/include/asm/pgtable.h +0 -1
@@ -12,7 +12,6 @@* Page table definitions for Qualcomm Hexagon processor. */ #include <asm/page.h> -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> /* A handy thing to have if one has the RAM. Declared in head.S */
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 4/14 ia64: add support for folded p4d page tables
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate, remove usage of __ARCH_USE_5LEVEL_HACK and replace 5level-fixup.h with pgtable-nop4d.h
--- arch/ia64/include/asm/pgalloc.h | 4 ++-- arch/ia64/include/asm/pgtable.h | 17 ++++++++--------- arch/ia64/mm/fault.c | 7 ++++++- arch/ia64/mm/hugetlbpage.c | 18 ++++++++++++------ arch/ia64/mm/init.c | 28 ++++++++++++++++++++++++---- 5 files changed, 52 insertions(+), 22 deletions(-)
arch/ia64/include/asm/pgalloc.h +2 -2
@@ -36,9 +36,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) #if CONFIG_PGTABLE_LEVELS = 4static inline void -pgd_populate (struct mm_struct *mm,pgd_t * pgd_entry,pud_t * pud) +p4d_populate (struct mm_struct *mm,p4d_t * p4d_entry,pud_t * pud) { -pgd_val (*pgd_entry) =__pa (pud); +p4d_val (*p4d_entry) =__pa (pud); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)
arch/ia64/include/asm/pgtable.h +8 -9
@@ -283,12 +283,12 @@ extern unsigned long VMALLOC_END;#define pud_page (pud) virt_to_page((pud_val(pud) + PAGE_OFFSET)) #if CONFIG_PGTABLE_LEVELS = 4 -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (!ia64_phys_addr_valid(pgd_val(pgd))) -#define pgd_present (pgd) (pgd_val(pgd) != 0UL) -#define pgd_clear (pgdp) (pgd_val(*(pgdp)) = 0UL) -#define pgd_page_vaddr (pgd) ((unsigned long) __va(pgd_val(pgd) & _PFN_MASK)) -#define pgd_page (pgd) virt_to_page((pgd_val(pgd) + PAGE_OFFSET)) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (!ia64_phys_addr_valid(p4d_val(p4d))) +#define p4d_present (p4d) (p4d_val(p4d) != 0UL) +#define p4d_clear (p4dp) (p4d_val(*(p4dp)) = 0UL) +#define p4d_page_vaddr (p4d) ((unsigned long) __va(p4d_val(p4d) & _PFN_MASK)) +#define p4d_page (p4d) virt_to_page((p4d_val(p4d) + PAGE_OFFSET))#endif /*
@@ -388,7 +388,7 @@ pgd_offset (const struct mm_struct *mm, unsigned long address) #if CONFIG_PGTABLE_LEVELS = 4/* Find an entry in the second-level page table.. */ #define pud_offset (dir,addr) \ - ((pud_t *) pgd_page_vaddr(*(dir)) + (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))) + ((pud_t *) p4d_page_vaddr(*(dir)) + (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#endif /* Find an entry in the third-level page table.. */
@@ -582,10 +582,9 @@ extern struct page *zero_page_memmap_ptr; #if CONFIG_PGTABLE_LEVELS = 3 -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopud.h> #endif -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> #include <asm-generic/pgtable.h> #endif /* _ASM_IA64_PGTABLE_H */
arch/ia64/mm/fault.c +6 -1
@@ -29,6 +29,7 @@ static int mapped_kernel_page_is_present (unsigned long address) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *ptep, pte;
@@ -37,7 +38,11 @@ mapped_kernel_page_is_present (unsigned long address)if (pgd_none (*pgd) ||pgd_bad (*pgd))return 0 ; - pud =pud_offset (pgd, address); + p4d =p4d_offset (pgd, address); +if (p4d_none (*p4d) ||p4d_bad (*p4d)) +return 0 ; + + pud =pud_offset (p4d, address);if (pud_none (*pud) ||pud_bad (*pud))return 0 ;
arch/ia64/mm/hugetlbpage.c +12 -6
@@ -30,12 +30,14 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) {unsigned long taddr =htlbpage_to_page (addr);pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte =NULL ; pgd =pgd_offset (mm, taddr); - pud =pud_alloc (mm, pgd, taddr); + p4d =p4d_offset (pgd, taddr); + pud =pud_alloc (mm, p4d, taddr);if (pud) { pmd =pmd_alloc (mm, pud, taddr);if (pmd)
@@ -49,17 +51,21 @@ huge_pte_offset (struct mm_struct *mm, unsigned long addr, unsigned long sz) {unsigned long taddr =htlbpage_to_page (addr);pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte =NULL ; pgd =pgd_offset (mm, taddr);if (pgd_present (*pgd)) { - pud =pud_offset (pgd, taddr); -if (pud_present (*pud)) { - pmd =pmd_offset (pud, taddr); -if (pmd_present (*pmd)) - pte =pte_offset_map (pmd, taddr); + p4d =p4d_offset (pgd, addr); +if (p4d_present (*p4d)) { + pud =pud_offset (p4d, taddr); +if (pud_present (*pud)) { + pmd =pmd_offset (pud, taddr); +if (pmd_present (*pmd)) + pte =pte_offset_map (pmd, taddr); + } } }
arch/ia64/mm/init.c +24 -4
@@ -208,6 +208,7 @@ static struct page * __init put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -215,7 +216,10 @@ put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot) pgd =pgd_offset_k (address);/* note: this is NOT pgd_offset()! */ { - pud =pud_alloc (&init_mm, pgd, address); + p4d =p4d_alloc (&init_mm, pgd, address); +if (!p4d) + goto out; + pud =pud_alloc (&init_mm, p4d, address);if (!pud) goto out; pmd =pmd_alloc (&init_mm, pud, address);
@@ -382,6 +386,7 @@ int vmemmap_find_next_valid_pfn(int node, int i) do { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte;
@@ -392,7 +397,13 @@ int vmemmap_find_next_valid_pfn(int node, int i)continue ; } - pud =pud_offset (pgd, end_address); + p4d =p4d_offset (pgd, end_address); +if (p4d_none (*p4d)) { + end_address +=P4D_SIZE ; +continue ; + } + + pud =pud_offset (p4d, end_address);if (pud_none (*pud)) { end_address +=PUD_SIZE ;continue ;
@@ -430,6 +441,7 @@ int __init create_mem_map_page_table(u64 start, u64 end, void *arg)struct page *map_start, *map_end;int node;pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -444,12 +456,20 @@ int __init create_mem_map_page_table(u64 start, u64 end, void *arg)for (address = start_page; address < end_page; address +=PAGE_SIZE ) { pgd =pgd_offset_k (address);if (pgd_none (*pgd)) { + p4d =memblock_alloc_node (PAGE_SIZE ,PAGE_SIZE , node); +if (!p4d) + goto err_alloc; +pgd_populate (&init_mm, pgd, p4d); + } + p4d =p4d_offset (pgd, address); + +if (p4d_none (*p4d)) { pud =memblock_alloc_node (PAGE_SIZE ,PAGE_SIZE , node);if (!pud) goto err_alloc; -pgd_populate (&init_mm, pgd, pud); +p4d_populate (&init_mm, p4d, pud); } - pud =pud_offset (pgd, address); + pud =pud_offset (p4d, address);if (pud_none (*pud)) { pmd =memblock_alloc_node (PAGE_SIZE ,PAGE_SIZE , node);
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 5/14 nios2: add support for folded p4d page tables
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and remove usage of __ARCH_USE_5LEVEL_HACK.
--- arch/nios2/include/asm/pgtable.h | 3 +-- arch/nios2/mm/fault.c | 9 +++++++-- arch/nios2/mm/ioremap.c | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-)
arch/nios2/include/asm/pgtable.h +1 -2
@@ -22,7 +22,6 @@#include <asm/tlbflush.h> #include <asm/pgtable-bits.h> -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #define FIRST_USER_ADDRESS 0UL
@@ -100,7 +99,7 @@ extern pte_t invalid_pte_table[PAGE_SIZE/sizeof(pte_t)];*/ static inline void set_pmd (pmd_t *pmdptr,pmd_t pmdval) { - pmdptr->pud.pgd.pgd = pmdval.pud.pgd.pgd; + *pmdptr = pmdval; }/* to find an entry in a page-table-directory */
arch/nios2/mm/fault.c +7 -2
@@ -245,6 +245,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,*/ int offset =pgd_index (address);pgd_t *pgd, *pgd_k; +p4d_t *p4d, *p4d_k;pud_t *pud, *pud_k;pmd_t *pmd, *pmd_k;pte_t *pte_k;
@@ -256,8 +257,12 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, goto no_context;set_pgd (pgd, *pgd_k); - pud =pud_offset (pgd, address); - pud_k =pud_offset (pgd_k, address); + p4d =p4d_offset (pgd, address); + p4d_k =p4d_offset (pgd_k, address); +if (!p4d_present (*p4d_k)) + goto no_context; + pud =pud_offset (p4d, address); + pud_k =pud_offset (p4d_k, address);if (!pud_present (*pud_k)) goto no_context; pmd =pmd_offset (pud, address);
arch/nios2/mm/ioremap.c +5 -1
@@ -86,11 +86,15 @@ static int remap_area_pages(unsigned long address, unsigned long phys_addr,if (address >= end)BUG (); do { + p4d_t *p4d; pud_t *pud; pmd_t *pmd; error = -ENOMEM; - pud = pud_alloc(&init_mm, dir, address); + p4d = p4d_alloc(&init_mm, dir, address); + if (!p4d) + break; + pud = pud_alloc(&init_mm, p4d, address); if (!pud) break; pmd = pmd_alloc(&init_mm, pud, address);
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 6/14 openrisc: add support for folded p4d page tables
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and remove usage of __ARCH_USE_5LEVEL_HACK.
--- arch/openrisc/include/asm/pgtable.h | 1 - arch/openrisc/mm/fault.c | 10 ++++++++-- arch/openrisc/mm/init.c | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-)
arch/openrisc/include/asm/pgtable.h +0 -1
@@ -21,7 +21,6 @@#ifndef __ASM_OPENRISC_PGTABLE_H#define __ASM_OPENRISC_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__
arch/openrisc/mm/fault.c +8 -2
@@ -296,6 +296,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,int offset =pgd_index (address);pgd_t *pgd, *pgd_k; +p4d_t *p4d, *p4d_k;pud_t *pud, *pud_k;pmd_t *pmd, *pmd_k;pte_t *pte_k;
@@ -322,8 +323,13 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,* it exists. */ - pud =pud_offset (pgd, address); - pud_k =pud_offset (pgd_k, address); + p4d =p4d_offset (pgd, address); + p4d_k =p4d_offset (pgd_k, address); +if (!p4d_present (*p4d_k)) + goto no_context; + + pud =pud_offset (p4d, address); + pud_k =pud_offset (p4d_k, address);if (!pud_present (*pud_k)) goto no_context;
arch/openrisc/mm/init.c +3 -1
@@ -71,6 +71,7 @@ static void __init map_ram(void)unsigned long v, p, e;pgprot_t prot;pgd_t *pge; +p4d_t *p4e;pud_t *pue;pmd_t *pme;pte_t *pte;
@@ -90,7 +91,8 @@ static void __init map_ram(void)while (p < e) {int j; - pue =pud_offset (pge, v); + p4e =p4d_offset (pge, v); + pue =pud_offset (p4e, v); pme =pmd_offset (pue, v);if ((u32 ) pue != (u32 ) pge || (u32 ) pme != (u32 ) pge) {
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 7/14 powerpc/32: drop get_pteptr()
In reply to Mike RapoportFrom: Christophe Leroy <[email protected]>
Commit 8d30c14cab30 ("powerpc/mm: Rework I$/D$ coherency (v3)") and commit 90ac19a8b21b ("[POWERPC] Abolish iopa(), mm_ptov(), io_block_mapping() from arch/powerpc") removed the use of get_pteptr() outside of mm/pgtable_32.c
In mm/pgtable_32.c, the only user of get_pteptr() is __change_page_attr() which operates on kernel context and on lowmem pages only.
Move page table traversal to __change_page_attr() and drop get_pteptr().
--- arch/powerpc/mm/pgtable_32.c | 43 ++++++------------------------------ 1 file changed, 7 insertions(+), 36 deletions(-)
arch/powerpc/mm/pgtable_32.c +7 -36
@@ -121,53 +121,24 @@ void __init mapin_ram(void) } } -/* Scan the real Linux page tables and return a PTE pointer for -* a virtual address in a context. -* Returns true (1) if PTE was found, zero otherwise. The pointer to -* the PTE pointer is unmodified if PTE is not found. -*/ -static int -get_pteptr (struct mm_struct *mm,unsigned long addr,pte_t **ptep,pmd_t **pmdp) -{ -pgd_t *pgd; -pud_t *pud; -pmd_t *pmd; -pte_t *pte; -int retval =0 ; - - pgd =pgd_offset (mm, addr &PAGE_MASK ); -if (pgd) { - pud =pud_offset (pgd, addr &PAGE_MASK ); -if (pud &&pud_present (*pud)) { - pmd =pmd_offset (pud, addr &PAGE_MASK ); -if (pmd_present (*pmd)) { - pte =pte_offset_map (pmd, addr &PAGE_MASK ); -if (pte) { - retval =1 ; - *ptep = pte; -if (pmdp) - *pmdp = pmd; -/* XXX caller needs to do pte_unmap, yuck */ - } - } - } - } -return (retval); -} -static int __change_page_attr_noflush (struct page *page,pgprot_t prot) {pte_t *kpte;pmd_t *kpmd; -unsigned long address; +unsigned long address, va;BUG_ON (PageHighMem (page)); address = (unsigned long )page_address (page); + va = address &PAGE_MASK ;if (v_block_mapped (address))return 0 ; -if (!get_pteptr (&init_mm, address, &kpte, &kpmd)) + + kpmd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); +if (!pmd_present (*kpmd))return -EINVAL ; + + kpte =pte_offset_map (kpmd, va);__set_pte_at (&init_mm, address, kpte,mk_pte (page, prot),0 );pte_unmap (kpte);
Signature · 1 line
2.24.0
Andrew Morton
maintainerRe: [PATCH v3 07/14] powerpc/32: drop get_pteptr()
Quoted 9 lines · On Thu, 27 Feb 2020 10:46:01 +0200 Mike Rapoport <[email protected]> wrote:
Commit
8d30c14cab30("powerpc/mm: Rework I$/D$ coherency (v3)") and commit90ac19a8b21b("[POWERPC] Abolishiopa(),mm_ptov(),io_block_mapping()fromarch/powerpc") removed the use ofget_pteptr()outside ofmm/pgtable_32.cIn
mm/pgtable_32.c, the only user ofget_pteptr()is__change_page_attr()which operates on kernel context and on lowmem pages only.Move page table traversal to
__change_page_attr()and dropget_pteptr().
People have been changing things in linux-next and the powerpc patches are hurting.
I'll disable this patch series for now. Can you please redo powerpc-32-drop-get_pteptr.patch and powerpc-add-support-for-folded-p4d-page-tables.patch (and powerpc-add-support-for-folded-p4d-page-tables-fix.patch)?
Thanks.
Mike Rapoport
Tested-by Christophe LeroyRe: [PATCH v3 07/14] powerpc/32: drop get_pteptr()
Quoted 12 lines · Andrew Morton
Quoted 9 lines · On Thu, 27 Feb 2020 10:46:01 +0200 Mike Rapoport <[email protected]> wrote:
Commit
8d30c14cab30("powerpc/mm: Rework I$/D$ coherency (v3)") and commit90ac19a8b21b("[POWERPC] Abolishiopa(),mm_ptov(),io_block_mapping()fromarch/powerpc") removed the use ofget_pteptr()outside ofmm/pgtable_32.cIn
mm/pgtable_32.c, the only user ofget_pteptr()is__change_page_attr()which operates on kernel context and on lowmem pages only.Move page table traversal to
__change_page_attr()and dropget_pteptr().
Quoted 7 lines
People have been changing things in linux-next and the powerpc patches are hurting.
I'll disable this patch series for now. Can you please redo powerpc-32-drop-get_pteptr.patch and powerpc-add-support-for-folded-p4d-page-tables.patch (and powerpc-add-support-for-folded-p4d-page-tables-fix.patch)?
This is the powerpc-add-support-for-folded-p4d-page-tables.patch on top of current powerpc/next. The powerpc-32-drop-get_pteptr.patch is already there and I've folded powerpc-add-support-for-folded-p4d-page-tables-fix.patch into this one.
From e2b405537d917c99430ee93b68fe4cb43d7b8787 Mon Sep 17 00:00:00 2001
From: Mike Rapoport <[email protected]>
Date: Sun, 24 Nov 2019 15:38:00 +0200
Subject: [PATCH v4] powerpc: add support for folded p4d page tables
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.
--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 60 ++++++++++--------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 10 ++-- arch/powerpc/kvm/book3s_64_mmu_radix.c | 32 ++++++---- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 26 +++++--- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 1 - arch/powerpc/mm/nohash/book3e_pgtable.c | 15 ++--- arch/powerpc/mm/pgtable.c | 30 ++++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 14 +++-- arch/powerpc/xmon/xmon.c | 18 +++--- 23 files changed, 197 insertions(+), 140 deletions(-)
arch/powerpc/include/asm/book3s/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h>
arch/powerpc/include/asm/book3s/64/hash.h +2 -2
@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) =0 ); +return (p4d_val (p4d) =0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );
arch/powerpc/include/asm/book3s/64/pgalloc.h +2 -2
@@ -85,9 +85,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ), pgd); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)
arch/powerpc/include/asm/book3s/64/pgtable.h +33 -27
120 lines not on this page · raw message
arch/powerpc/include/asm/book3s/64/radix.h +3 -3
@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)
@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE
arch/powerpc/include/asm/nohash/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__
arch/powerpc/include/asm/nohash/64/pgalloc.h +1 -1
@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) {
arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16 -16
@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record
@@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) = 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) = 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \
arch/powerpc/include/asm/nohash/64/pgtable.h +3 -3
@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/*
arch/powerpc/include/asm/pgtable.h +5 -5
@@ -44,12 +44,12 @@ struct mm_struct;#ifdef CONFIG_PPC32 static inline pmd_t *pmd_ptr (struct mm_struct *mm,unsigned long va) { -return pmd_offset (pud_offset (pgd_offset (mm, va), va), va); +return pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, va), va), va), va); }static inline pmd_t *pmd_ptr_k (unsigned long va) { -return pmd_offset (pud_offset (pgd_offset_k (va), va), va); +return pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va); }static inline pte_t *virt_to_kpte (unsigned long vaddr)
@@ -156,9 +156,9 @@ static inline bool pud_is_leaf(pud_t pud) }#endif -#ifndef pgd_is_leaf -#define pgd_is_leaf pgd_is_leaf -static inline bool pgd_is_leaf (pgd_t pgd) +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) {return false; }
arch/powerpc/kvm/book3s_64_mmu_radix.c +19 -13
@@ -499,13 +499,14 @@ void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, unsigned int lpid)unsigned long ig;for (ig =0 ; ig <PTRS_PER_PGD ; ++ig, ++pgd) { +p4d_t *p4d =p4d_offset (pgd,0 );pud_t *pud; -if (!pgd_present (*pgd)) +if (!p4d_present (*p4d))continue ; - pud =pud_offset (pgd,0 ); + pud =pud_offset (p4d,0 );kvmppc_unmap_free_pud (kvm, pud, lpid); -pgd_clear (pgd); +p4d_clear (p4d); } }
@@ -566,6 +567,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long *rmapp, struct rmap_nested **n_rmap) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud, *new_pud =NULL ;pmd_t *pmd, *new_pmd =NULL ;pte_t *ptep, *new_ptep =NULL ;
@@ -573,9 +575,11 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Traverse the guest's 2nd-level tree, allocate new levels needed */ pgd = pgtable +pgd_index (gpa); + p4d =p4d_offset (pgd, gpa); + pud =NULL ; -if (pgd_present (*pgd)) - pud =pud_offset (pgd, gpa); +if (p4d_present (*p4d)) + pud =pud_offset (p4d, gpa);else new_pud =pud_alloc_one (kvm->mm, gpa);
@@ -596,13 +600,13 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Now traverse again under the lock and change the tree */ ret = -ENOMEM ; -if (pgd_none (*pgd)) { +if (p4d_none (*p4d)) {if (!new_pud) goto out_unlock; -pgd_populate (kvm->mm, pgd, new_pud); +p4d_populate (kvm->mm, p4d, new_pud); new_pud =NULL ; } - pud =pud_offset (pgd, gpa); + pud =pud_offset (p4d, gpa);if (pud_is_leaf (*pud)) {unsigned long hgpa = gpa &PUD_MASK ;
@@ -1219,7 +1223,8 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,unsigned long gpa;pgd_t *pgt;struct kvm_nested_guest *nested; -pgd_t pgd, *pgdp; +pgd_t *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ptep;
@@ -1292,13 +1297,14 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf, } pgdp = pgt +pgd_index (gpa); - pgd =READ_ONCE (*pgdp); -if (!(pgd_val (pgd) & _PAGE_PRESENT)) { - gpa = (gpa &PGDIR_MASK ) +PGDIR_SIZE ; + p4dp =p4d_offset (pgdp, gpa); + p4d =READ_ONCE (*p4dp); +if (!(p4d_val (p4d) & _PAGE_PRESENT)) { + gpa = (gpa &P4D_MASK ) +P4D_SIZE ;continue ; } - pudp =pud_offset (&pgd, gpa); + pudp =pud_offset (&p4d, gpa); pud =READ_ONCE (*pudp);if (!(pud_val (pud) & _PAGE_PRESENT)) { gpa = (gpa &PUD_MASK ) +PUD_SIZE ;
arch/powerpc/lib/code-patching.c +6 -1
@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;
arch/powerpc/mm/book3s64/hash_pgtable.c +3 -1
@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >H_PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
arch/powerpc/mm/book3s64/radix_pgtable.c +17 -9
@@ -64,17 +64,19 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea); -if (pgd_none (*pgdp)) { + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size =PUD_SIZE ) { ptep = (pte_t *)pudp; goto set_the_pte;
@@ -114,6 +116,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -136,7 +139,8 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,* boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;if (map_page_size =PUD_SIZE ) {
@@ -173,6 +177,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -185,7 +190,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end,for (idx = start; idx < end; idx +=PAGE_SIZE ) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;if (pud_is_leaf (*pudp)) {
@@ -847,6 +853,7 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end)unsigned long addr, next;pud_t *pud_base;pgd_t *pgd; +p4d_t *p4d;spin_lock (&init_mm.page_table_lock);
@@ -854,15 +861,16 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end) next =pgd_addr_end (addr, end); pgd =pgd_offset_k (addr); -if (!pgd_present (*pgd)) + p4d =p4d_offset (pgd, addr); +if (!p4d_present (*p4d))continue ; -if (pgd_is_leaf (*pgd)) { -split_kernel_mapping (addr, end,PGDIR_SIZE , (pte_t *)pgd); +if (p4d_is_leaf (*p4d)) { +split_kernel_mapping (addr, end,P4D_SIZE , (pte_t *)p4d);continue ; } - pud_base = (pud_t *)pgd_page_vaddr (*pgd); + pud_base = (pud_t *)p4d_page_vaddr (*p4d);remove_pud_table (pud_base, addr, next); }
arch/powerpc/mm/book3s64/subpage_prot.c +4 -2
@@ -54,15 +54,17 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;spinlock_t *ptl; pgd =pgd_offset (mm, addr); -if (pgd_none (*pgd)) + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d))return ; - pud =pud_offset (pgd, addr); + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ; pmd =pmd_offset (pud, addr);
arch/powerpc/mm/hugetlbpage.c +16 -12
@@ -112,6 +112,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;pmd_t *pm;hugepd_t *hpdp =NULL ;
@@ -121,20 +122,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift =PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift =PUD_SHIFT )
@@ -159,10 +161,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz#else if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {
@@ -384,7 +386,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) {
@@ -394,7 +396,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) { if (pud_none_or_clear_bad(pud))
@@ -429,8 +431,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,if (end -1 > ceiling -1 )return ; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm); }
@@ -443,6 +445,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/*
@@ -465,10 +468,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*
@@ -481,7 +485,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);
arch/powerpc/mm/kasan/kasan_init_32.c +0 -1
@@ -37,7 +37,6 @@ static int __init kasan_init_shadow_page_tables(unsigned long k_start, unsignedpte_t *new =NULL ; pmd =pmd_ptr_k (k_start); -for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) { k_next =pgd_addr_end (k_cur, k_end);if ((void *)pmd_page_vaddr (*pmd) != kasan_early_shadow_pte)
arch/powerpc/mm/nohash/book3e_pgtable.c +8 -7
@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -80,7 +81,8 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
@@ -91,13 +93,12 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)return -ENOMEM ; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDED -if (pgd_none (*pgdp)) { - pudp =early_alloc_pgtable (PUD_TABLE_SIZE ); -pgd_populate (&init_mm, pgdp, pudp); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );pud_populate (&init_mm, pudp, pmdp);
arch/powerpc/mm/pgtable.c +19 -11
@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,void assert_pte_locked (struct mm_struct *mm,unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)return ; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);/*
@@ -312,12 +315,13 @@ EXPORT_SYMBOL_GPL(vmalloc_to_phys);pte_t *__find_linux_pte (pgd_t *pgdir,unsigned long ea,bool *is_thp,unsigned *hpage_shift) { -pgd_t pgd, *pgdp; +pgd_t *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;hugepd_t *hpdp =NULL ; -unsigned pdshift =PGDIR_SHIFT ; +unsigned pdshift;if (hpage_shift) *hpage_shift =0 ;
@@ -325,24 +329,28 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,if (is_thp) *is_thp = false; - pgdp = pgdir +pgd_index (ea); - pgd =READ_ONCE (*pgdp);/* * Always operate on the local stack value. This make sure the * value don't get updated by a parallel THP split/collapse, * page fault or a page unmap. The return pte_t * is still not * stable. So should be checked there for above conditions. +* Top level is an exception because it is folded into p4d. */ -if (pgd_none (pgd)) + pgdp = pgdir +pgd_index (ea); + p4dp =p4d_offset (pgdp, ea); + p4d =READ_ONCE (*p4dp); + pdshift =P4D_SHIFT ; + +if (p4d_none (p4d))return NULL ; -if (pgd_is_leaf (pgd)) { - ret_pte = (pte_t *)pgdp; +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; goto out; } -if (is_hugepd (__hugepd (pgd_val (pgd)))) { - hpdp = (hugepd_t *)&pgd; +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; goto out_huge; }
@@ -352,7 +360,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,* irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);if (pud_none (pud))
arch/powerpc/mm/pgtable_64.c +5 -5
@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif
arch/powerpc/mm/ptdump/hashpagetable.c +17 -3
@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {pgd_t *pgd =pgd_offset_k (0UL );
@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }
arch/powerpc/mm/ptdump/ptdump.c +8 -6
@@ -277,9 +277,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -304,11 +304,13 @@ static void walk_pagetables(struct pg_state *st)* the hash pagetable. */ for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) { -if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd)) -/* pgd exists */ -walk_pud (st, pgd, addr); +p4d_t *p4d =p4d_offset (pgd,0 ); + +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr);else -note_page (st, addr,1 ,pgd_val (*pgd)); +note_page (st, addr,1 ,p4d_val (*p4d)); } }
arch/powerpc/xmon/xmon.c +10 -8
@@ -3130,6 +3130,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -3161,20 +3162,21 @@ static void show_pte(unsigned long addr) pgdir =pgd_offset (mm,0 ); } -if (pgd_none (*pgdp)) { -printf ("no linux page table for address\n" ); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" );return ; } -printf ("pgd @ 0x%px\n" , pgdir); - -if (pgd_is_leaf (*pgdp)) { -format_pte (pgdp,pgd_val (*pgdp)); +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp));return ; } -printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" );
Signature · 1 line
2.25.1
Mike Rapoport
reviewerPATCH v3 8/14 powerpc: add support for folded p4d page tables
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.
--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 60 ++++++++++--------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 6 +- arch/powerpc/kvm/book3s_64_mmu_radix.c | 30 ++++++---- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 26 +++++--- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 ++--- arch/powerpc/mm/pgtable.c | 30 ++++++---- arch/powerpc/mm/pgtable_32.c | 4 +- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 14 +++-- arch/powerpc/xmon/xmon.c | 18 +++--- 28 files changed, 207 insertions(+), 149 deletions(-)
arch/powerpc/include/asm/book3s/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h>
arch/powerpc/include/asm/book3s/64/hash.h +2 -2
@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) =0 ); +return (p4d_val (p4d) =0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );
arch/powerpc/include/asm/book3s/64/pgalloc.h +2 -2
@@ -85,9 +85,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ), pgd); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)
arch/powerpc/include/asm/book3s/64/pgtable.h +33 -27
120 lines not on this page · raw message
arch/powerpc/include/asm/book3s/64/radix.h +3 -3
@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)
@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE
arch/powerpc/include/asm/nohash/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__
arch/powerpc/include/asm/nohash/64/pgalloc.h +1 -1
@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) {
arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16 -16
@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record
@@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) = 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) = 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \
arch/powerpc/include/asm/nohash/64/pgtable.h +3 -3
@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/*
arch/powerpc/include/asm/pgtable.h +3 -3
@@ -139,9 +139,9 @@ static inline bool pud_is_leaf(pud_t pud) }#endif -#ifndef pgd_is_leaf -#define pgd_is_leaf pgd_is_leaf -static inline bool pgd_is_leaf (pgd_t pgd) +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) {return false; }
arch/powerpc/kvm/book3s_64_mmu_radix.c +18 -12
@@ -499,13 +499,14 @@ void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, unsigned int lpid)unsigned long ig;for (ig =0 ; ig <PTRS_PER_PGD ; ++ig, ++pgd) { +p4d_t *p4d =p4d_offset (pgd,0 );pud_t *pud; -if (!pgd_present (*pgd)) +if (!p4d_present (*p4d))continue ; - pud =pud_offset (pgd,0 ); + pud =pud_offset (p4d,0 );kvmppc_unmap_free_pud (kvm, pud, lpid); -pgd_clear (pgd); +p4d_clear (p4d); } }
@@ -566,6 +567,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long *rmapp, struct rmap_nested **n_rmap) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud, *new_pud =NULL ;pmd_t *pmd, *new_pmd =NULL ;pte_t *ptep, *new_ptep =NULL ;
@@ -573,9 +575,11 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Traverse the guest's 2nd-level tree, allocate new levels needed */ pgd = pgtable +pgd_index (gpa); + p4d =p4d_offset (pgd, gpa); + pud =NULL ; -if (pgd_present (*pgd)) - pud =pud_offset (pgd, gpa); +if (p4d_present (*p4d)) + pud =pud_offset (p4d, gpa);else new_pud =pud_alloc_one (kvm->mm, gpa);
@@ -596,13 +600,13 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Now traverse again under the lock and change the tree */ ret = -ENOMEM ; -if (pgd_none (*pgd)) { +if (p4d_none (*p4d)) {if (!new_pud) goto out_unlock; -pgd_populate (kvm->mm, pgd, new_pud); +p4d_populate (kvm->mm, p4d, new_pud); new_pud =NULL ; } - pud =pud_offset (pgd, gpa); + pud =pud_offset (p4d, gpa);if (pud_is_leaf (*pud)) {unsigned long hgpa = gpa &PUD_MASK ;
@@ -1220,6 +1224,7 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,pgd_t *pgt;struct kvm_nested_guest *nested;pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ptep;
@@ -1292,13 +1297,14 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf, } pgdp = pgt +pgd_index (gpa); - pgd =READ_ONCE (*pgdp); -if (!(pgd_val (pgd) & _PAGE_PRESENT)) { - gpa = (gpa &PGDIR_MASK ) +PGDIR_SIZE ; + p4dp =p4d_offset (pgdp, gpa); + p4d =READ_ONCE (*p4dp); +if (!(p4d_val (p4d) & _PAGE_PRESENT)) { + gpa = (gpa &P4D_MASK ) +P4D_SIZE ;continue ; } - pudp =pud_offset (&pgd, gpa); + pudp =pud_offset (&p4d, gpa); pud =READ_ONCE (*pudp);if (!(pud_val (pud) & _PAGE_PRESENT)) { gpa = (gpa &PUD_MASK ) +PUD_SIZE ;
arch/powerpc/lib/code-patching.c +6 -1
@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;
arch/powerpc/mm/book3s32/mmu.c +1 -1
@@ -312,7 +312,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea)if (!Hash)return ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, ea), ea), ea); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, ea), ea), ea), ea);if (!pmd_none (*pmd))add_hash_page (mm->context.id, ea,pmd_val (*pmd)); }
arch/powerpc/mm/book3s32/tlb.c +2 -2
@@ -87,7 +87,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start,if (start >= end)return ; end = (end -1 ) | ~PAGE_MASK ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, start), start), start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, start), start), start), start);for (;;) { pmd_end = ((start +PGDIR_SIZE ) &PGDIR_MASK ) -1 ;if (pmd_end > end)
@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)return ; } mm = (vmaddr <TASK_SIZE )? vma->vm_mm: &init_mm; - pmd =pmd_offset (pud_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr), vmaddr);if (!pmd_none (*pmd))flush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 ); }
arch/powerpc/mm/book3s64/hash_pgtable.c +3 -1
@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >H_PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
arch/powerpc/mm/book3s64/radix_pgtable.c +17 -9
@@ -64,17 +64,19 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea); -if (pgd_none (*pgdp)) { + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size =PUD_SIZE ) { ptep = (pte_t *)pudp; goto set_the_pte;
@@ -114,6 +116,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -136,7 +139,8 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,* boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;if (map_page_size =PUD_SIZE ) {
@@ -173,6 +177,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -185,7 +190,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end,for (idx = start; idx < end; idx +=PAGE_SIZE ) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;if (pud_is_leaf (*pudp)) {
@@ -847,6 +853,7 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end)unsigned long addr, next;pud_t *pud_base;pgd_t *pgd; +p4d_t *p4d;spin_lock (&init_mm.page_table_lock);
@@ -854,15 +861,16 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end) next =pgd_addr_end (addr, end); pgd =pgd_offset_k (addr); -if (!pgd_present (*pgd)) + p4d =p4d_offset (pgd, addr); +if (!p4d_present (*p4d))continue ; -if (pgd_is_leaf (*pgd)) { -split_kernel_mapping (addr, end,PGDIR_SIZE , (pte_t *)pgd); +if (p4d_is_leaf (*p4d)) { +split_kernel_mapping (addr, end,P4D_SIZE , (pte_t *)p4d);continue ; } - pud_base = (pud_t *)pgd_page_vaddr (*pgd); + pud_base = (pud_t *)p4d_page_vaddr (*p4d);remove_pud_table (pud_base, addr, next); }
arch/powerpc/mm/book3s64/subpage_prot.c +4 -2
@@ -54,15 +54,17 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;spinlock_t *ptl; pgd =pgd_offset (mm, addr); -if (pgd_none (*pgd)) + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d))return ; - pud =pud_offset (pgd, addr); + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ; pmd =pmd_offset (pud, addr);
arch/powerpc/mm/hugetlbpage.c +16 -12
@@ -119,6 +119,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;pmd_t *pm;hugepd_t *hpdp =NULL ;
@@ -128,20 +129,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift =PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift =PUD_SHIFT )
@@ -166,10 +168,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz#else if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {
@@ -390,7 +392,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) {
@@ -400,7 +402,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) { if (pud_none_or_clear_bad(pud))
@@ -435,8 +437,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,if (end -1 > ceiling -1 )return ; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm); }
@@ -449,6 +451,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/*
@@ -471,10 +474,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*
@@ -487,7 +491,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);
arch/powerpc/mm/kasan/kasan_init_32.c +4 -4
@@ -36,7 +36,7 @@ static int __init kasan_init_shadow_page_tables(unsigned long k_start, unsignedunsigned long k_cur, k_next;pte_t *new =NULL ; - pmd =pmd_offset (pud_offset (pgd_offset_k (k_start), k_start), k_start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_start), k_start), k_start), k_start);for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) { k_next =pgd_addr_end (k_cur, k_end);
@@ -78,7 +78,7 @@ static int __init kasan_init_region(void *start, size_t size) block =memblock_alloc (k_end - k_start,PAGE_SIZE );for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);void *va = block + k_cur - k_start;pte_t pte =pfn_pte (PHYS_PFN (__pa (va)),PAGE_KERNEL );
@@ -102,7 +102,7 @@ static void __init kasan_remap_early_shadow_ro(void)kasan_populate_pte (kasan_early_shadow_pte, prot);for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);pte_t *ptep =pte_offset_kernel (pmd, k_cur);if ((pte_val (*ptep) &PTE_RPN_MASK ) != pa)
@@ -201,7 +201,7 @@ void __init kasan_early_init(void)unsigned long addr =KASAN_SHADOW_START ;unsigned long end =KASAN_SHADOW_END ;unsigned long next; -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (addr), addr), addr); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (addr), addr), addr), addr);BUILD_BUG_ON (KASAN_SHADOW_START & ~PGDIR_MASK );
arch/powerpc/mm/mem.c +2 -2
@@ -69,8 +69,8 @@ EXPORT_SYMBOL(kmap_prot);static inline pte_t *virt_to_kpte (unsigned long vaddr) { -return pte_offset_kernel (pmd_offset (pud_offset (pgd_offset_k (vaddr), - vaddr), vaddr), vaddr); +return pte_offset_kernel (pmd_offset (pud_offset (p4d_offset (pgd_offset_k (vaddr), + vaddr), vaddr), vaddr), vaddr); }#endif
arch/powerpc/mm/nohash/40x.c +2 -2
@@ -104,7 +104,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val);
@@ -119,7 +119,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp =__pmd (val); v +=LARGE_PAGE_SIZE_4M ;
arch/powerpc/mm/nohash/book3e_pgtable.c +8 -7
@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -80,7 +81,8 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
@@ -91,13 +93,12 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)return -ENOMEM ; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDED -if (pgd_none (*pgdp)) { - pudp =early_alloc_pgtable (PUD_TABLE_SIZE ); -pgd_populate (&init_mm, pgdp, pudp); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );pud_populate (&init_mm, pudp, pmdp);
arch/powerpc/mm/pgtable.c +19 -11
@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,void assert_pte_locked (struct mm_struct *mm,unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)return ; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);/*
@@ -312,12 +315,13 @@ EXPORT_SYMBOL_GPL(vmalloc_to_phys);pte_t *__find_linux_pte (pgd_t *pgdir,unsigned long ea,bool *is_thp,unsigned *hpage_shift) { -pgd_t pgd, *pgdp; +pgd_t *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;hugepd_t *hpdp =NULL ; -unsigned pdshift =PGDIR_SHIFT ; +unsigned pdshift;if (hpage_shift) *hpage_shift =0 ;
@@ -325,24 +329,28 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,if (is_thp) *is_thp = false; - pgdp = pgdir +pgd_index (ea); - pgd =READ_ONCE (*pgdp);/* * Always operate on the local stack value. This make sure the * value don't get updated by a parallel THP split/collapse, * page fault or a page unmap. The return pte_t * is still not * stable. So should be checked there for above conditions. +* Top level is an exception because it is folded into p4d. */ -if (pgd_none (pgd)) + pgdp = pgdir +pgd_index (ea); + p4dp =p4d_offset (pgdp, ea); + p4d =READ_ONCE (*p4dp); + pdshift =P4D_SHIFT ; + +if (p4d_none (p4d))return NULL ; -if (pgd_is_leaf (pgd)) { - ret_pte = (pte_t *)pgdp; +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; goto out; } -if (is_hugepd (__hugepd (pgd_val (pgd)))) { - hpdp = (hugepd_t *)&pgd; +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; goto out_huge; }
@@ -352,7 +360,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,* irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);if (pud_none (pud))
arch/powerpc/mm/pgtable_32.c +2 -2
@@ -63,7 +63,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot)int err = -ENOMEM ;/* Use upper 10 bits of VA to index the first level map */ - pd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + pd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);/* Use middle 10 bits of VA to index the second-level map */ if (likely (slab_is_available ())) pg =pte_alloc_kernel (pd, va);
@@ -134,7 +134,7 @@ static int __change_page_attr_noflush(struct page *page, pgprot_t prot)if (v_block_mapped (address))return 0 ; - kpmd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + kpmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);if (!pmd_present (*kpmd))return -EINVAL ;
arch/powerpc/mm/pgtable_64.c +5 -5
@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif
arch/powerpc/mm/ptdump/hashpagetable.c +17 -3
@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {pgd_t *pgd =pgd_offset_k (0UL );
@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }
arch/powerpc/mm/ptdump/ptdump.c +8 -6
@@ -277,9 +277,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -304,11 +304,13 @@ static void walk_pagetables(struct pg_state *st)* the hash pagetable. */ for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) { -if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd)) -/* pgd exists */ -walk_pud (st, pgd, addr); +p4d_t *p4d =p4d_offset (pgd,0 ); + +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr);else -note_page (st, addr,1 ,pgd_val (*pgd)); +note_page (st, addr,1 ,p4d_val (*p4d)); } }
arch/powerpc/xmon/xmon.c +10 -8
@@ -3130,6 +3130,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -3161,20 +3162,21 @@ static void show_pte(unsigned long addr) pgdir =pgd_offset (mm,0 ); } -if (pgd_none (*pgdp)) { -printf ("no linux page table for address\n" ); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" );return ; } -printf ("pgd @ 0x%px\n" , pgdir); - -if (pgd_is_leaf (*pgdp)) { -format_pte (pgdp,pgd_val (*pgdp)); +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp));return ; } -printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" );
Signature · 1 line
2.24.0
kbuild test robot
build botRe: [PATCH v3 08/14] powerpc: add support for folded p4d page tables
I love your patch! Yet something to improve:
Hi Mike,
I love your patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on arm/for-next arm64/for-next/core linus/master v5.6-rc3 next-20200228] [cannot apply to mmotm/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Mike-Rapoport/mm-remove-__ARCH_HAS_5LEVEL_HACK/20200227-171638 base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next config: powerpc-ppc64_defconfig (attached as .config) compiler: powerpc64-linux-gcc (GCC) 7.5.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.5.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag
All errors (new ones prefixed by >>):
arch/powerpc/kvm/book3s_64_mmu_radix.c: In function 'debugfs_radix_read':
>> arch/powerpc/kvm/book3s_64_mmu_radix.c:1226:8: error: unused variable 'pgd' [-Werror=unused-variable]
pgd_t pgd, *pgdp;
^~~cc1: all warnings being treated as errors
vim +/pgd +1226 arch/powerpc/kvm/book3s_64_mmu_radix.c
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1215
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1216 static ssize_t debugfs_radix_read(struct file *file, char __user *buf,
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1217 size_t len, loff_t *ppos)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1218 {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1219 struct debugfs_radix_state *p = file->private_data;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1220 ssize_t ret, r;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1221 unsigned long n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1222 struct kvm *kvm;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1223 unsigned long gpa;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1224 pgd_t *pgt;
83a055104eaf89 Paul Mackerras 2018-10-08 1225 struct kvm_nested_guest *nested;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 @1226 pgd_t pgd, *pgdp;
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1227 p4d_t p4d, *p4dp;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1228 pud_t pud, *pudp;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1229 pmd_t pmd, *pmdp;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1230 pte_t *ptep;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1231 int shift;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1232 unsigned long pte;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1233
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1234 kvm = p->kvm;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1235 if (!kvm_is_radix(kvm))
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1236 return 0;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1237
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1238 ret = mutex_lock_interruptible(&p->mutex);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1239 if (ret)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1240 return ret;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1241
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1242 if (p->chars_left) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1243 n = p->chars_left;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1244 if (n > len)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1245 n = len;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1246 r = copy_to_user(buf, p->buf + p->buf_index, n);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1247 n -= r;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1248 p->chars_left -= n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1249 p->buf_index += n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1250 buf += n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1251 len -= n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1252 ret = n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1253 if (r) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1254 if (!n)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1255 ret = -EFAULT;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1256 goto out;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1257 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1258 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1259
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1260 gpa = p->gpa;
83a055104eaf89 Paul Mackerras 2018-10-08 1261 nested = NULL;
83a055104eaf89 Paul Mackerras 2018-10-08 1262 pgt = NULL;
83a055104eaf89 Paul Mackerras 2018-10-08 1263 while (len != 0 && p->lpid >= 0) {
83a055104eaf89 Paul Mackerras 2018-10-08 1264 if (gpa >= RADIX_PGTABLE_RANGE) {
83a055104eaf89 Paul Mackerras 2018-10-08 1265 gpa = 0;
83a055104eaf89 Paul Mackerras 2018-10-08 1266 pgt = NULL;
83a055104eaf89 Paul Mackerras 2018-10-08 1267 if (nested) {
83a055104eaf89 Paul Mackerras 2018-10-08 1268 kvmhv_put_nested(nested);
83a055104eaf89 Paul Mackerras 2018-10-08 1269 nested = NULL;
83a055104eaf89 Paul Mackerras 2018-10-08 1270 }
83a055104eaf89 Paul Mackerras 2018-10-08 1271 p->lpid = kvmhv_nested_next_lpid(kvm, p->lpid);
83a055104eaf89 Paul Mackerras 2018-10-08 1272 p->hdr = 0;
83a055104eaf89 Paul Mackerras 2018-10-08 1273 if (p->lpid < 0)
83a055104eaf89 Paul Mackerras 2018-10-08 1274 break;
83a055104eaf89 Paul Mackerras 2018-10-08 1275 }
83a055104eaf89 Paul Mackerras 2018-10-08 1276 if (!pgt) {
83a055104eaf89 Paul Mackerras 2018-10-08 1277 if (p->lpid == 0) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1278 pgt = kvm->arch.pgtable;
83a055104eaf89 Paul Mackerras 2018-10-08 1279 } else {
83a055104eaf89 Paul Mackerras 2018-10-08 1280 nested = kvmhv_get_nested(kvm, p->lpid, false);
83a055104eaf89 Paul Mackerras 2018-10-08 1281 if (!nested) {
83a055104eaf89 Paul Mackerras 2018-10-08 1282 gpa = RADIX_PGTABLE_RANGE;
83a055104eaf89 Paul Mackerras 2018-10-08 1283 continue;
83a055104eaf89 Paul Mackerras 2018-10-08 1284 }
83a055104eaf89 Paul Mackerras 2018-10-08 1285 pgt = nested->shadow_pgtable;
83a055104eaf89 Paul Mackerras 2018-10-08 1286 }
83a055104eaf89 Paul Mackerras 2018-10-08 1287 }
83a055104eaf89 Paul Mackerras 2018-10-08 1288 n = 0;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1289 if (!p->hdr) {
83a055104eaf89 Paul Mackerras 2018-10-08 1290 if (p->lpid > 0)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1291 n = scnprintf(p->buf, sizeof(p->buf),
83a055104eaf89 Paul Mackerras 2018-10-08 1292 "\nNested LPID %d: ", p->lpid);
83a055104eaf89 Paul Mackerras 2018-10-08 1293 n += scnprintf(p->buf + n, sizeof(p->buf) - n,
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1294 "pgdir: %lx\n", (unsigned long)pgt);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1295 p->hdr = 1;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1296 goto copy;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1297 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1298
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1299 pgdp = pgt + pgd_index(gpa);
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1300 p4dp = p4d_offset(pgdp, gpa);
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1301 p4d = READ_ONCE(*p4dp);
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1302 if (!(p4d_val(p4d) & _PAGE_PRESENT)) {
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1303 gpa = (gpa & P4D_MASK) + P4D_SIZE;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1304 continue;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1305 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1306
f97ba1f7bf6bd7 Mike Rapoport 2020-02-27 1307 pudp = pud_offset(&p4d, gpa);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1308 pud = READ_ONCE(*pudp);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1309 if (!(pud_val(pud) & _PAGE_PRESENT)) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1310 gpa = (gpa & PUD_MASK) + PUD_SIZE;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1311 continue;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1312 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1313 if (pud_val(pud) & _PAGE_PTE) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1314 pte = pud_val(pud);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1315 shift = PUD_SHIFT;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1316 goto leaf;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1317 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1318
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1319 pmdp = pmd_offset(&pud, gpa);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1320 pmd = READ_ONCE(*pmdp);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1321 if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1322 gpa = (gpa & PMD_MASK) + PMD_SIZE;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1323 continue;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1324 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1325 if (pmd_val(pmd) & _PAGE_PTE) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1326 pte = pmd_val(pmd);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1327 shift = PMD_SHIFT;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1328 goto leaf;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1329 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1330
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1331 ptep = pte_offset_kernel(&pmd, gpa);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1332 pte = pte_val(READ_ONCE(*ptep));
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1333 if (!(pte & _PAGE_PRESENT)) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1334 gpa += PAGE_SIZE;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1335 continue;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1336 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1337 shift = PAGE_SHIFT;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1338 leaf:
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1339 n = scnprintf(p->buf, sizeof(p->buf),
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1340 " %lx: %lx %d\n", gpa, pte, shift);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1341 gpa += 1ul << shift;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1342 copy:
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1343 p->chars_left = n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1344 if (n > len)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1345 n = len;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1346 r = copy_to_user(buf, p->buf, n);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1347 n -= r;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1348 p->chars_left -= n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1349 p->buf_index = n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1350 buf += n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1351 len -= n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1352 ret += n;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1353 if (r) {
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1354 if (!ret)
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1355 ret = -EFAULT;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1356 break;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1357 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1358 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1359 p->gpa = gpa;
83a055104eaf89 Paul Mackerras 2018-10-08 1360 if (nested)
83a055104eaf89 Paul Mackerras 2018-10-08 1361 kvmhv_put_nested(nested);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1362
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1363 out:
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1364 mutex_unlock(&p->mutex);
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1365 return ret;
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1366 }
9a94d3ee2d1599 Paul Mackerras 2018-10-08 1367 :::::: The code at line 1226 was first introduced by commit
:::::: 9a94d3ee2d159927c0f8e5078228eadbce8dda43 KVM: PPC: Book3S HV: Add a debugfs file to dump radix mappings
:::::: TO: Paul Mackerras <[email protected]>
:::::: CC: Michael Ellerman <[email protected]>
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Mike Rapoport
reviewerPATCH v3 9/14 sh: fault: Modernize printing of kernel messages
In reply to Mike RapoportFrom: Geert Uytterhoeven <[email protected]>
- Convert from printk() to pr_*(),
- Add missing continuations,
- Use "%llx" to format u64,
- Join multiple prints in show_fault_oops() into a single print.
--- arch/sh/mm/fault.c | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-)
arch/sh/mm/fault.c +18 -21
@@ -47,10 +47,10 @@ static void show_pte(struct mm_struct *mm, unsigned long addr) pgd = swapper_pg_dir; } -printk (KERN_ALERT "pgd = %p\n" , pgd); +pr_alert ("pgd = %p\n" , pgd); pgd +=pgd_index (addr); -printk (KERN_ALERT "[%08lx] *pgd=%0*Lx" , addr, - (u32)(sizeof (*pgd) *2 ), (u64 )pgd_val (*pgd)); +pr_alert ("[%08lx] *pgd=%0*llx" , addr, (u32)(sizeof (*pgd) *2 ), + (u64 )pgd_val (*pgd)); do { pud_t *pud;
@@ -61,33 +61,33 @@ static void show_pte(struct mm_struct *mm, unsigned long addr)break ;if (pgd_bad (*pgd)) { -printk ("(bad)" ); +pr_cont ("(bad)" );break ; } pud =pud_offset (pgd, addr);if (PTRS_PER_PUD !=1 ) -printk (", *pud=%0*Lx" , (u32)(sizeof (*pud) *2 ), - (u64 )pud_val (*pud)); +pr_cont (", *pud=%0*llx" , (u32)(sizeof (*pud) *2 ), + (u64 )pud_val (*pud));if (pud_none (*pud))break ;if (pud_bad (*pud)) { -printk ("(bad)" ); +pr_cont ("(bad)" );break ; } pmd =pmd_offset (pud, addr);if (PTRS_PER_PMD !=1 ) -printk (", *pmd=%0*Lx" , (u32)(sizeof (*pmd) *2 ), - (u64 )pmd_val (*pmd)); +pr_cont (", *pmd=%0*llx" , (u32)(sizeof (*pmd) *2 ), + (u64 )pmd_val (*pmd));if (pmd_none (*pmd))break ;if (pmd_bad (*pmd)) { -printk ("(bad)" ); +pr_cont ("(bad)" );break ; }
@@ -96,11 +96,11 @@ static void show_pte(struct mm_struct *mm, unsigned long addr)break ; pte =pte_offset_kernel (pmd, addr); -printk (", *pte=%0*Lx" , (u32)(sizeof (*pte) *2 ), - (u64 )pte_val (*pte)); +pr_cont (", *pte=%0*llx" , (u32)(sizeof (*pte) *2 ), + (u64 )pte_val (*pte)); }while (0 ); -printk ("\n" ); +pr_cont ("\n" ); }static inline pmd_t *vmalloc_sync_one (pgd_t *pgd,unsigned long address)
@@ -188,14 +188,11 @@ show_fault_oops(struct pt_regs *regs, unsigned long address)if (!oops_may_print ())return ; -printk (KERN_ALERT "BUG: unable to handle kernel " ); -if (address <PAGE_SIZE ) -printk (KERN_CONT "NULL pointer dereference" ); -else -printk (KERN_CONT "paging request" ); - -printk (KERN_CONT " at %08lx\n" , address); -printk (KERN_ALERT "PC:" ); +pr_alert ("BUG: unable to handle kernel %s at %08lx\n" , + address <PAGE_SIZE ?"NULL pointer dereference" + :"paging request" , + address); +pr_alert ("PC:" );printk_address (regs->pc,1 );show_pte (NULL , address);
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 10/14 sh: drop __pXd_offset() macros that duplicate pXd_index() ones
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
The __pXd_offset() macros are identical to the pXd_index() macros and there is no point to keep both of them. All architectures define and use pXd_index() so let's keep only those to make mips consistent with the rest of the kernel.
--- arch/sh/include/asm/pgtable_32.h | 5 ++--- arch/sh/include/asm/pgtable_64.h | 5 ++--- arch/sh/mm/init.c | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-)
arch/sh/include/asm/pgtable_32.h +2 -3
@@ -407,13 +407,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)/* to find an entry in a page-table-directory. */ #define pgd_index (address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))#define pgd_offset (mm, address) ((mm)->pgd + pgd_index(address)) -#define __pgd_offset (address) pgd_index(address)/* to find an entry in a kernel page-table-directory */ #define pgd_offset_k (address) pgd_offset(&init_mm, address) -#define __pud_offset (address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define __pmd_offset (address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) +#define pud_index (address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) +#define pmd_index (address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))/* Find an entry in the third-level page table.. */ #define pte_index (address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
arch/sh/include/asm/pgtable_64.h +2 -3
@@ -46,14 +46,13 @@ static __inline__ void set_pte(pte_t *pteptr, pte_t pteval)/* To find an entry in a generic PGD. */ #define pgd_index (address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define __pgd_offset (address) pgd_index(address)#define pgd_offset (mm, address) ((mm)->pgd+pgd_index(address))/* To find an entry in a kernel PGD. */ #define pgd_offset_k (address) pgd_offset(&init_mm, address) -#define __pud_offset (address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define __pmd_offset (address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) +#define pud_index (address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) +/* #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) */ /* * PMD level access routines. Same notes as above.
arch/sh/mm/init.c +3 -3
@@ -172,9 +172,9 @@ void __init page_table_range_init(unsigned long start, unsigned long end,unsigned long vaddr; vaddr = start; - i =__pgd_offset (vaddr); - j =__pud_offset (vaddr); - k =__pmd_offset (vaddr); + i =pgd_index (vaddr); + j =pud_index (vaddr); + k =pmd_index (vaddr); pgd = pgd_base + i;for ( ; (i <PTRS_PER_PGD ) && (vaddr != end); pgd++, i++) {
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 11/14 sh: add support for folded p4d page tables
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and remove usage of __ARCH_USE_5LEVEL_HACK.
--- arch/sh/include/asm/pgtable-2level.h | 1 - arch/sh/include/asm/pgtable-3level.h | 1 - arch/sh/kernel/io_trapped.c | 7 ++++++- arch/sh/mm/cache-sh4.c | 4 +++- arch/sh/mm/cache-sh5.c | 7 ++++++- arch/sh/mm/fault.c | 26 +++++++++++++++++++++++--- arch/sh/mm/hugetlbpage.c | 28 ++++++++++++++++++---------- arch/sh/mm/init.c | 9 ++++++++- arch/sh/mm/kmap.c | 2 +- arch/sh/mm/tlbex_32.c | 6 +++++- arch/sh/mm/tlbex_64.c | 7 ++++++- 11 files changed, 76 insertions(+), 22 deletions(-)
arch/sh/include/asm/pgtable-2level.h +0 -1
@@ -2,7 +2,6 @@#ifndef __ASM_SH_PGTABLE_2LEVEL_H#define __ASM_SH_PGTABLE_2LEVEL_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> /*
arch/sh/include/asm/pgtable-3level.h +0 -1
@@ -2,7 +2,6 @@#ifndef __ASM_SH_PGTABLE_3LEVEL_H#define __ASM_SH_PGTABLE_3LEVEL_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopud.h> /*
arch/sh/kernel/io_trapped.c +6 -1
@@ -136,6 +136,7 @@ EXPORT_SYMBOL_GPL(match_trapped_io_handler);static struct trapped_io *lookup_tiop (unsigned long address) {pgd_t *pgd_k; +p4d_t *p4d_k;pud_t *pud_k;pmd_t *pmd_k;pte_t *pte_k;
@@ -145,7 +146,11 @@ static struct trapped_io *lookup_tiop(unsigned long address)if (!pgd_present (*pgd_k))return NULL ; - pud_k =pud_offset (pgd_k, address); + p4d_k =p4d_offset (pgd_k, address); +if (!p4d_present (*p4d_k)) +return NULL ; + + pud_k =pud_offset (p4d_k, address);if (!pud_present (*pud_k))return NULL ;
arch/sh/mm/cache-sh4.c +3 -1
@@ -209,6 +209,7 @@ static void sh4_flush_cache_page(void *args)unsigned long address, pfn, phys;int map_coherent =0 ;pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -224,7 +225,8 @@ static void sh4_flush_cache_page(void *args)return ; pgd =pgd_offset (vma->vm_mm, address); - pud =pud_offset (pgd, address); + p4d =p4d_offset (pgd, address); + pud =pud_offset (p4d, address); pmd =pmd_offset (pud, address); pte =pte_offset_kernel (pmd, address);
arch/sh/mm/cache-sh5.c +6 -1
@@ -383,6 +383,7 @@ static void sh64_dcache_purge_user_pages(struct mm_struct *mm, unsigned long addr, unsigned long end) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -397,7 +398,11 @@ static void sh64_dcache_purge_user_pages(struct mm_struct *mm,if (pgd_bad (*pgd))return ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d) ||p4d_bad (*p4d)) +return ; + + pud =pud_offset (p4d, addr);if (pud_none (*pud) ||pud_bad (*pud))return ;
arch/sh/mm/fault.c +23 -3
@@ -53,6 +53,7 @@ static void show_pte(struct mm_struct *mm, unsigned long addr) (u64)pgd_val(*pgd)); do { + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte;
@@ -65,7 +66,20 @@ static void show_pte(struct mm_struct *mm, unsigned long addr)break ; } - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (PTRS_PER_P4D !=1 ) +pr_cont (", *p4d=%0*Lx" , (u32)(sizeof (*p4d) *2 ), + (u64 )p4d_val (*p4d)); + +if (p4d_none (*p4d)) +break ; + +if (p4d_bad (*p4d)) { +pr_cont ("(bad)" ); +break ; + } + + pud =pud_offset (p4d, addr);if (PTRS_PER_PUD !=1 )pr_cont (", *pud=%0*llx" , (u32)(sizeof (*pud) *2 ), (u64 )pud_val (*pud));
@@ -107,6 +121,7 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) {unsigned index =pgd_index (address);pgd_t *pgd_k; +p4d_t *p4d, *p4d_k;pud_t *pud, *pud_k;pmd_t *pmd, *pmd_k;
@@ -116,8 +131,13 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)if (!pgd_present (*pgd_k))return NULL ; - pud =pud_offset (pgd, address); - pud_k =pud_offset (pgd_k, address); + p4d =p4d_offset (pgd, address); + p4d_k =p4d_offset (pgd_k, address); +if (!p4d_present (*p4d_k)) +return NULL ; + + pud =pud_offset (p4d, address); + pud_k =pud_offset (p4d_k, address);if (!pud_present (*pud_k))return NULL ;
arch/sh/mm/hugetlbpage.c +18 -10
@@ -26,17 +26,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte =NULL ; pgd =pgd_offset (mm, addr);if (pgd) { - pud =pud_alloc (mm, pgd, addr); -if (pud) { - pmd =pmd_alloc (mm, pud, addr); -if (pmd) - pte =pte_alloc_map (mm, pmd, addr); + p4d =p4d_alloc (mm, pgd, addr); +if (p4d) { + pud =pud_alloc (mm, p4d, addr); +if (pud) { + pmd =pmd_alloc (mm, pud, addr); +if (pmd) + pte =pte_alloc_map (mm, pmd, addr); + } } }
@@ -47,17 +51,21 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte =NULL ; pgd =pgd_offset (mm, addr);if (pgd) { - pud =pud_offset (pgd, addr); -if (pud) { - pmd =pmd_offset (pud, addr); -if (pmd) - pte =pte_offset_map (pmd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d) { + pud =pud_offset (p4d, addr); +if (pud) { + pmd =pmd_offset (pud, addr); +if (pmd) + pte =pte_offset_map (pmd, addr); + } } }
arch/sh/mm/init.c +8 -1
@@ -45,6 +45,7 @@ void __init __weak plat_mem_setup(void)static pte_t *__get_pte_phys (unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -54,7 +55,13 @@ static pte_t *__get_pte_phys(unsigned long addr)return NULL ; } - pud =pud_alloc (NULL , pgd, addr); + p4d =p4d_alloc (NULL , pgd, addr); +if (unlikely (!p4d)) { +p4d_ERROR (*p4d); +return NULL ; + } + + pud =pud_alloc (NULL , p4d, addr);if (unlikely (!pud)) {pud_ERROR (*pud);return NULL ;
arch/sh/mm/kmap.c +1 -1
@@ -15,7 +15,7 @@#include <asm/cacheflush.h> #define kmap_get_fixmap_pte (vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr)) + pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr)), vaddr)static pte_t *kmap_coherent_pte;
arch/sh/mm/tlbex_32.c +5 -1
@@ -23,6 +23,7 @@ handle_tlbmiss(struct pt_regs *regs, unsigned long error_code, unsigned long address) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -42,7 +43,10 @@ handle_tlbmiss(struct pt_regs *regs, unsigned long error_code, pgd =pgd_offset (current->mm, address); } - pud =pud_offset (pgd, address); + p4d =p4d_offset (pgd, address); +if (p4d_none_or_clear_bad (p4d)) +return 1 ; + pud =pud_offset (p4d, address);if (pud_none_or_clear_bad (pud))return 1 ; pmd =pmd_offset (pud, address);
arch/sh/mm/tlbex_64.c +6 -1
@@ -44,6 +44,7 @@ static int handle_tlbmiss(unsigned long long protection_flags, unsigned long address) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -58,7 +59,11 @@ static int handle_tlbmiss(unsigned long long protection_flags, pgd =pgd_offset (current->mm, address); } - pud =pud_offset (pgd, address); + p4d =p4d_offset (pgd, address); +if (p4d_none (*p4d) || !p4d_present (*p4d)) +return 1 ; + + pud =pud_offset (p4d, address);if (pud_none (*pud) || !pud_present (*pud))return 1 ;
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 12/14 unicore32: remove __ARCH_USE_5LEVEL_HACK
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
The unicore32 architecture has 2 level page tables and asm-generic/pgtable-nopmd.h and explicit casts from pud_t to pgd_t for page table folding.
Add p4d walk in the only place that actually unfolds the pud level and remove __ARCH_USE_5LEVEL_HACK.
--- arch/unicore32/include/asm/pgtable.h | 1 - arch/unicore32/kernel/hibernate.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-)
arch/unicore32/include/asm/pgtable.h +0 -1
@@ -9,7 +9,6 @@#ifndef __UNICORE_PGTABLE_H__#define __UNICORE_PGTABLE_H__ -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/cpu-single.h>
arch/unicore32/kernel/hibernate.c +3 -1
@@ -33,9 +33,11 @@ struct swsusp_arch_regs swsusp_arch_regs_cpu0;static pmd_t *resume_one_md_table_init (pgd_t *pgd) {pud_t *pud; +p4d_t *p4d;pmd_t *pmd_table; - pud =pud_offset (pgd,0 ); + p4d =p4d_offset (pgd,0 ); + pud =pud_offset (p4d,0 ); pmd_table =pmd_offset (pud,0 );return pmd_table;
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 13/14 asm-generic: remove pgtable-nop4d-hack.h
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
No architecture defines __ARCH_USE_5LEVEL_HACK and therefore pgtable-nop4d-hack.h will be never actually included.
Remove it.
--- include/asm-generic/pgtable-nop4d-hack.h | 64 ------------------------ include/asm-generic/pgtable-nopud.h | 4 -- 2 files changed, 68 deletions(-) delete mode 100644 include/asm-generic/pgtable-nop4d-hack.h
include/asm-generic/pgtable-nop4d-hack.h deleted +0 -64
@@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _PGTABLE_NOP4D_HACK_H -#define _PGTABLE_NOP4D_HACK_H - -#ifndef __ASSEMBLY__ -#include <asm-generic/5level-fixup.h> - -#define __PAGETABLE_PUD_FOLDED 1 - -/* -* Having the pud type consist of a pgd gets the size right, and allows -* us to conceptually access the pgd entry that this pud is folded into -* without casting. -*/ -typedef struct {pgd_t pgd; }pud_t ; - -#define PUD_SHIFT PGDIR_SHIFT -#define PTRS_PER_PUD 1 -#define PUD_SIZE (1UL << PUD_SHIFT) -#define PUD_MASK (~(PUD_SIZE-1)) - -/* -* The "pgd_xxx()" functions here are trivial for a folded two-level -* setup: the pud is never bad, and a pud always exists (as it's folded -* into the pgd entry) -*/ -static inline int pgd_none (pgd_t pgd) {return 0 ; } -static inline int pgd_bad (pgd_t pgd) {return 0 ; } -static inline int pgd_present (pgd_t pgd) {return 1 ; } -static inline void pgd_clear (pgd_t *pgd) { } -#define pud_ERROR (pud) (pgd_ERROR((pud).pgd)) - -#define pgd_populate (mm, pgd, pud) do { } while (0) -#define pgd_populate_safe (mm, pgd, pud) do { } while (0) -/* -* (puds are folded into pgds so this doesn't get actually called, -* but the define is needed for a generic inline function.) -*/ -#define set_pgd (pgdptr, pgdval) set_pud((pud_t *)(pgdptr), (pud_t) { pgdval }) - -static inline pud_t *pud_offset (pgd_t *pgd,unsigned long address) -{ -return (pud_t *)pgd; -} - -#define pud_val (x) (pgd_val((x).pgd)) -#define __pud (x) ((pud_t) { __pgd(x) }) - -#define pgd_page (pgd) (pud_page((pud_t){ pgd })) -#define pgd_page_vaddr (pgd) (pud_page_vaddr((pud_t){ pgd })) - -/* -* allocating and freeing a pud is trivial: the 1-entry pud is -* inside the pgd, so has no extra memory associated with it. -*/ -#define pud_alloc_one (mm, address) NULL -#define pud_free (mm, x) do { } while (0) -#define __pud_free_tlb (tlb, x, a) do { } while (0) - -#undef pud_addr_end -#define pud_addr_end (addr, end) (end) - -#endif /* __ASSEMBLY__ */ -#endif /* _PGTABLE_NOP4D_HACK_H */
include/asm-generic/pgtable-nopud.h +0 -4
@@ -4,9 +4,6 @@#ifndef __ASSEMBLY__ -#ifdef __ARCH_USE_5LEVEL_HACK -#include <asm-generic/pgtable-nop4d-hack.h> -#else #include <asm-generic/pgtable-nop4d.h> #define __PAGETABLE_PUD_FOLDED 1
@@ -65,5 +62,4 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)#define pud_addr_end (addr, end) (end)#endif /* __ASSEMBLY__ */ -#endif /* !__ARCH_USE_5LEVEL_HACK */ #endif /* _PGTABLE_NOPUD_H */
Signature · 1 line
2.24.0
Mike Rapoport
reviewerPATCH v3 14/14 mm: remove __ARCH_HAS_5LEVEL_HACK and include/asm-generic/5level-fixup.h
In reply to Mike RapoportFrom: Mike Rapoport <[email protected]>
There are no architectures that use include/asm-generic/5level-fixup.h therefore it can be removed along with __ARCH_HAS_5LEVEL_HACK define and the code it surrounds
--- include/asm-generic/5level-fixup.h | 58 ------------------------------ include/linux/mm.h | 6 ---- mm/kasan/init.c | 11 ------ mm/memory.c | 8 ----- 4 files changed, 83 deletions(-) delete mode 100644 include/asm-generic/5level-fixup.h
include/asm-generic/5level-fixup.h deleted +0 -58
@@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _5LEVEL_FIXUP_H -#define _5LEVEL_FIXUP_H - -#define __ARCH_HAS_5LEVEL_HACK -#define __PAGETABLE_P4D_FOLDED 1 - -#define P4D_SHIFT PGDIR_SHIFT -#define P4D_SIZE PGDIR_SIZE -#define P4D_MASK PGDIR_MASK -#define MAX_PTRS_PER_P4D 1 -#define PTRS_PER_P4D 1 - -#define p4d_t pgd_t - -#define pud_alloc (mm, p4d, address) \ - ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \ - NULL : pud_offset(p4d, address)) - -#define p4d_alloc (mm, pgd, address) (pgd) -#define p4d_offset (pgd, start) (pgd) - -#ifndef __ASSEMBLY__ -static inline int p4d_none (p4d_t p4d) -{ -return 0 ; -} - -static inline int p4d_bad (p4d_t p4d) -{ -return 0 ; -} - -static inline int p4d_present (p4d_t p4d) -{ -return 1 ; -} -#endif - -#define p4d_ERROR (p4d) do { } while (0) -#define p4d_clear (p4d) pgd_clear(p4d) -#define p4d_val (p4d) pgd_val(p4d) -#define p4d_populate (mm, p4d, pud) pgd_populate(mm, p4d, pud) -#define p4d_populate_safe (mm, p4d, pud) pgd_populate(mm, p4d, pud) -#define p4d_page (p4d) pgd_page(p4d) -#define p4d_page_vaddr (p4d) pgd_page_vaddr(p4d) - -#define __p4d (x) __pgd(x) -#define set_p4d (p4dp, p4d) set_pgd(p4dp, p4d) - -#undef p4d_free_tlb -#define p4d_free_tlb (tlb, x, addr) do { } while (0) -#define p4d_free (mm, x) do { } while (0) - -#undef p4d_addr_end -#define p4d_addr_end (addr, end) (end) - -#endif
include/linux/mm.h +0 -6
@@ -1841,11 +1841,6 @@ int __pte_alloc_kernel(pmd_t *pmd);#if defined(CONFIG_MMU ) -/* - * The following ifdef needed to get the 5level-fixup.h header to work. - * Remove it when 5level-fixup.h has been removed. - */ -#ifndef __ARCH_HAS_5LEVEL_HACKstatic inline p4d_t *p4d_alloc (struct mm_struct *mm,pgd_t *pgd,unsigned long address) {
@@ -1859,7 +1854,6 @@ static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,return (unlikely (p4d_none (*p4d)) &&__pud_alloc (mm, p4d, address)) ?NULL :pud_offset (p4d, address); } -#endif /* !__ARCH_HAS_5LEVEL_HACK */ static inline pmd_t *pmd_alloc (struct mm_struct *mm,pud_t *pud,unsigned long address) {
mm/kasan/init.c +0 -11
@@ -250,20 +250,9 @@ int __ref kasan_populate_early_shadow(const void *shadow_start,* 3,2 - level page tables where we don't have * puds,pmds, so pgd_populate(), pud_populate() * is noops. -* -* The ifndef is required to avoid build breakage. -* -* With 5level-fixup.h, pgd_populate() is not nop and -* we reference kasan_early_shadow_p4d. It's not defined -* unless 5-level paging enabled. -* -* The ifndef can be dropped once all KASAN-enabled -* architectures will switch to pgtable-nop4d.h. */ -#ifndef __ARCH_HAS_5LEVEL_HACKpgd_populate (&init_mm, pgd,lm_alias (kasan_early_shadow_p4d)); -#endif p4d =p4d_offset (pgd, addr);p4d_populate (&init_mm, p4d,lm_alias (kasan_early_shadow_pud));
mm/memory.c +0 -8
@@ -4252,19 +4252,11 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)smp_wmb ();/* See comment in __pte_alloc */ spin_lock (&mm->page_table_lock); -#ifndef __ARCH_HAS_5LEVEL_HACKif (!p4d_present (*p4d)) {mm_inc_nr_puds (mm);p4d_populate (mm, p4d, new); }else /* Another has populated it */ pud_free (mm, new); -#else -if (!pgd_present (*p4d)) { -mm_inc_nr_puds (mm); -pgd_populate (mm, p4d, new); - }else /* Another has populated it */ -pud_free (mm, new); -#endif /* __ARCH_HAS_5LEVEL_HACK */ spin_unlock (&mm->page_table_lock);return 0 ; }
Signature · 1 line
2.24.0