THP support for PPC64
Subsystems (12)
cover THP support for PPC64
Hi,
This patchset adds transparent hugepage support for PPC64.
TODO:
* powernv still doesn't boot
* hash preload support in update_mmu_cache_pmd
Some numbers:
The latency measurements code from Anton found at
http://ozlabs.org/~anton/junkcode/latency2001.c
THP disabled 64K page size
------------------------
[root@llmp24l02 ~]# ./latency2001 8G
8589934592 731.73 cycles 205.77 ns
[root@llmp24l02 ~]# ./latency2001 8G
8589934592 743.39 cycles 209.05 ns
[root@llmp24l02 ~]#
THP disabled large page via hugetlbfs
-------------------------------------
[root@llmp24l02 ~]# ./latency2001 -l 8G
8589934592 416.09 cycles 117.01 ns
[root@llmp24l02 ~]# ./latency2001 -l 8G
8589934592 415.74 cycles 116.91 ns
THP enabled 64K page size.
----------------
[root@llmp24l02 ~]# ./latency2001 8G
8589934592 405.07 cycles 113.91 ns
[root@llmp24l02 ~]# ./latency2001 8G
8589934592 411.82 cycles 115.81 ns
[root@llmp24l02 ~]#
We are close to hugetlbfs in latency and we can achieve this with zero config/page reservation. Most of the allocations above are fault allocated. I haven't really measured the collapse alloc impact.
Another test that does 50000000 random access over 1GB area goes from 2.65 seconds to 1.07 seconds with this patchset.
Change from V2:
* Change patch "powerpc: Reduce PTE table memory wastage" to use much simpler approach for PTE page sharing.
* Changes to handle huge pages in KVM code.
* Address other review comments
Changes from V1
* Address review comments
* More patch split
* Add batch hpte invalidate for hugepages.
Changes from RFC V2:
* Address review comments
* More code cleanup and patch split
Changes from RFC V1:
* HugeTLB fs now works
* Compile issues fixed
* rebased to v3.8
* Patch series reorded so that ppc64 cleanups and MM THP changes are moved early in the series. This should help in picking those patches early.
Thanks,
-aneesh
PATCH 1/25 powerpc: Use signed formatting when printing error
From: "Aneesh Kumar K.V" <[email protected]>
PAPR defines these errors as negative values. So print them accordingly for easy debugging.
--- arch/powerpc/platforms/pseries/lpar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/powerpc/platforms/pseries/lpar.c +1 -1
@@ -155,7 +155,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,*/ if (unlikely (lpar_rc !=H_SUCCESS )) {if (!(vflags &HPTE_V_BOLTED )) -pr_devel (" lpar err %lu\n" , lpar_rc); +pr_devel (" lpar err %ld\n" , lpar_rc);return -2 ; }if (!(vflags &HPTE_V_BOLTED ))
Signature · 1 line
1.7.10
PATCH 2/25 powerpc: Save DAR and DSISR in pt_regs on MCE
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We were not saving DAR and DSISR on MCE. Save then and also print the values along with exception details in xmon.
--- arch/powerpc/kernel/exceptions-64s.S | 9 +++++++++ arch/powerpc/xmon/xmon.c | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-)
arch/powerpc/kernel/exceptions-64s.S +9 -0
@@ -640,9 +640,18 @@ slb_miss_user_pseries: .align 7 .globl machine_check_common machine_check_common: + + mfspr r10,SPRN_DAR + std r10,PACA_EXGEN+EX_DAR(r13) + mfspr r10,SPRN_DSISR + stw r10,PACA_EXGEN+EX_DSISR(r13) EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC) FINISH_NAP DISABLE_INTS + ld r3,PACA_EXGEN+EX_DAR(r13) + lwz r4,PACA_EXGEN+EX_DSISR(r13) + std r3,_DAR(r1) + std r4,_DSISR(r1) bl .save_nvgprs addi r3,r1,STACK_FRAME_OVERHEAD bl .machine_check_exception
arch/powerpc/xmon/xmon.c +1 -1
@@ -1423,7 +1423,7 @@ static void excprint(struct pt_regs *fp)printf (" sp: %lx\n" , fp->gpr[1 ]);printf (" msr: %lx\n" , fp->msr); -if (trap ==0x300 || trap ==0x380 || trap ==0x600 ) { +if (trap ==0x300 || trap ==0x380 || trap ==0x600 || trap ==0x200 ) {printf (" dar: %lx\n" , fp->dar);if (trap !=0x380 )printf (" dsisr: %lx\n" , fp->dsisr);
Signature · 1 line
1.7.10
PATCH 3/25 powerpc: Don't hard code the size of pte page
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
USE PTRS_PER_PTE to indicate the size of pte page. To support THP, later patches will be changing PTRS_PER_PTE value.
--- arch/powerpc/include/asm/pgtable.h | 6 ++++++ arch/powerpc/mm/hash_low_64.S | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-)
arch/powerpc/include/asm/pgtable.h +6 -0
@@ -17,6 +17,12 @@ struct mm_struct;# include <asm/pgtable-ppc32.h> #endif +/* +* We save the slot number & secondary bit in the second half of the +* PTE page. We use the 8 bytes per each pte entry. +*/ +#define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8) +#ifndef __ASSEMBLY__#include <asm/tlbflush.h>
arch/powerpc/mm/hash_low_64.S +2 -2
@@ -490,7 +490,7 @@ END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) beq htab_inval_old_hpte ld r6,STK_PARAM(R6)(r1) - ori r26,r6,0x8000 /* Load the hidx mask */ + ori r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */ ld r26,0(r26) addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
@@ -607,7 +607,7 @@ htab_pte_insert_ok: sld r4,r4,r5 andc r26,r26,r4 or r26,r26,r3 - ori r5,r6,0x8000 + ori r5,r6,PTE_PAGE_HIDX_OFFSET std r26,0(r5) lwsync std r30,0(r6)
Signature · 1 line
1.7.10
PATCH 4/25 powerpc: Reduce the PTE_INDEX_SIZE
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
This make one PMD cover 16MB range. That helps in easier implementation of THP on power. THP core code make use of one pmd entry to track the hugepage and the range mapped by a single pmd entry should be equal to the hugepage size supported by the hardware.
--- arch/powerpc/include/asm/pgtable-ppc64-64k.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/powerpc/include/asm/pgtable-ppc64-64k.h +2 -2
@@ -4,10 +4,10 @@#include <asm-generic/pgtable-nopud.h> -#define PTE_INDEX_SIZE 12 +#define PTE_INDEX_SIZE 8#define PMD_INDEX_SIZE 12#define PUD_INDEX_SIZE 0 -#define PGD_INDEX_SIZE 6 +#define PGD_INDEX_SIZE 10#ifndef __ASSEMBLY__#define PTE_TABLE_SIZE (sizeof(real_pte_t) << PTE_INDEX_SIZE)
Signature · 1 line
1.7.10
PATCH 5/25 powerpc: Move the pte free routines from common header
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
This patch moves the common code to 32/64 bit headers and also duplicate 4K_PAGES and 64K_PAGES section. We will later change the 64 bit 64K_PAGES version to support smaller PTE fragments. The patch doesn't introduce any functional changes.
--- arch/powerpc/include/asm/pgalloc-32.h | 45 ++++++++++ arch/powerpc/include/asm/pgalloc-64.h | 157 ++++++++++++++++++++++++++++++--- arch/powerpc/include/asm/pgalloc.h | 46 +--------- 3 files changed, 189 insertions(+), 59 deletions(-)
arch/powerpc/include/asm/pgalloc-32.h +45 -0
@@ -37,6 +37,17 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr);extern pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long addr); +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) +{ +free_page ((unsigned long )pte); +} + +static inline void pte_free (struct mm_struct *mm,pgtable_t ptepage) +{ +pgtable_page_dtor (ptepage); +__free_page (ptepage); +} +static inline void pgtable_free (void *table,unsigned index_size) {BUG_ON (index_size);/* 32-bit doesn't use this */
@@ -45,4 +56,38 @@ static inline void pgtable_free(void *table, unsigned index_size)#define check_pgt_cache () do { } while (0) +#ifdef CONFIG_SMP +static inline void pgtable_free_tlb (struct mmu_gather *tlb, +void *table,int shift) +{ +unsigned long pgf = (unsigned long )table; +BUG_ON (shift >MAX_PGTABLE_INDEX_SIZE ); + pgf |= shift; +tlb_remove_table (tlb, (void *)pgf); +} + +static inline void __tlb_remove_table (void *_table) +{ +void *table = (void *)((unsigned long )_table & ~MAX_PGTABLE_INDEX_SIZE ); +unsigned shift = (unsigned long )_table &MAX_PGTABLE_INDEX_SIZE ; + +pgtable_free (table, shift); +} +#else +static inline void pgtable_free_tlb (struct mmu_gather *tlb, +void *table,int shift) +{ +pgtable_free (table, shift); +} +#endif + +static inline void __pte_free_tlb (struct mmu_gather *tlb,pgtable_t table, +unsigned long address) +{ +struct page *page =page_address (table); + +tlb_flush_pgtable (tlb, address); +pgtable_page_dtor (page); +pgtable_free_tlb (tlb, page,0 ); +}#endif /* _ASM_POWERPC_PGALLOC_32_H */
arch/powerpc/include/asm/pgalloc-64.h +143 -14
194 lines not on this page · raw message
arch/powerpc/include/asm/pgalloc.h +1 -45
63 lines not on this page · raw message
Signature · 1 line
1.7.10
PATCH 6/25 powerpc: Reduce PTE table memory wastage
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We allocate one page for the last level of linux page table. With THP and large page size of 16MB, that would mean we are wasting large part of that page. To map 16MB area, we only need a PTE space of 2K with 64K page size. This patch reduce the space wastage by sharing the page allocated for the last level of linux page table with multiple pmd entries. We call these smaller chunks PTE page fragments and allocated page, PTE page.
In order to support systems which doesn't have 64K HPTE support, we also add another 2K to PTE page fragment. The second half of the PTE fragments is used for storing slot and secondary bit information of an HPTE. With this we now have a 4K PTE fragment.
We use a simple approach to share the PTE page. On allocation, we bump the PTE page refcount to 16 and share the PTE page with the next 16 pte alloc request. This should help in the node locality of the PTE page fragment, assuming that the immediate pte alloc request will mostly come from the same NUMA node. We don't try to reuse the freed PTE page fragment. Hence we could be waisting some space.
--- arch/powerpc/include/asm/mmu-book3e.h | 4 + arch/powerpc/include/asm/mmu-hash64.h | 4 + arch/powerpc/include/asm/page.h | 4 + arch/powerpc/include/asm/pgalloc-64.h | 72 ++++------------- arch/powerpc/kernel/setup_64.c | 4 +- arch/powerpc/mm/mmu_context_hash64.c | 35 ++++++++ arch/powerpc/mm/pgtable_64.c | 144 +++++++++++++++++++++++++++++++++ 7 files changed, 209 insertions(+), 58 deletions(-)
arch/powerpc/include/asm/mmu-book3e.h +4 -0
@@ -231,6 +231,10 @@ typedef struct {u64 high_slices_psize;/* 4 bits per slice for now */ u16 user_psize;/* page size index */ #endif +#ifdef CONFIG_PPC_64K_PAGES +/* for 4K PTE fragment support */ +struct page *pgtable_page; +#endif } mm_context_t;/* Page size definitions, common between 32 and 64-bit
arch/powerpc/include/asm/mmu-hash64.h +4 -0
@@ -498,6 +498,10 @@ typedef struct {unsigned long acop;/* mask of enabled coprocessor types */ unsigned int cop_pid;/* pid value used with coprocessors */ #endif /* CONFIG_PPC_ICSWX */ +#ifdef CONFIG_PPC_64K_PAGES +/* for 4K PTE fragment support */ +struct page *pgtable_page; +#endif } mm_context_t;
arch/powerpc/include/asm/page.h +4 -0
@@ -378,7 +378,11 @@ void arch_free_page(struct page *page, int order);struct vm_area_struct ; +#ifdef CONFIG_PPC_64K_PAGES +typedef pte_t *pgtable_t ; +#else typedef struct page *pgtable_t ; +#endif #include <asm-generic/memory_model.h> #endif /* __ASSEMBLY__ */
arch/powerpc/include/asm/pgalloc-64.h +15 -57
118 lines not on this page · raw message
arch/powerpc/kernel/setup_64.c +3 -1
@@ -575,7 +575,9 @@ void __init setup_arch(char **cmdline_p) init_mm.end_code = (unsigned long ) _etext; init_mm.end_data = (unsigned long ) _edata; init_mm.brk = klimit; - +#ifdef CONFIG_PPC_64K_PAGES + init_mm.context.pgtable_page =NULL ; +#endif irqstack_early_init ();exc_lvl_early_init ();emergency_stack_init ();
arch/powerpc/mm/mmu_context_hash64.c +35 -0
@@ -86,6 +86,9 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)spin_lock_init (mm->context.cop_lockp);#endif /* CONFIG_PPC_ICSWX */ +#ifdef CONFIG_PPC_64K_PAGES + mm->context.pgtable_page =NULL ; +#endif return 0 ; }
@@ -97,13 +100,45 @@ void __destroy_context(int context_id) }EXPORT_SYMBOL_GPL (__destroy_context); +#ifdef CONFIG_PPC_64K_PAGES +static void destroy_pagetable_page (struct mm_struct *mm) +{ +int count; +struct page *page; + + page = mm->context.pgtable_page; +if (!page) +return ; + +/* drop all the pending references */ + count =atomic_read (&page->_mapcount) +1 ; +/* We allow PTE_FRAG_NR(16) fragments from a PTE page */ + count =atomic_sub_return (16 - count, &page->_count); +if (!count) { +pgtable_page_dtor (page); +reset_page_mapcount (page); +free_hot_cold_page (page,0 ); + } +} + +#else +static inline void destroy_pagetable_page (struct mm_struct *mm) +{ +return ; +} +#endif + +void destroy_context (struct mm_struct *mm) { +#ifdef CONFIG_PPC_ICSWX drop_cop (mm->context.acop, mm);kfree (mm->context.cop_lockp); mm->context.cop_lockp =NULL ;#endif /* CONFIG_PPC_ICSWX */ + +destroy_pagetable_page (mm);__destroy_context (mm->context.id);subpage_prot_free (mm); mm->context.id =MMU_NO_CONTEXT ;
arch/powerpc/mm/pgtable_64.c +144 -0
147 lines not on this page · raw message
Signature · 1 line
1.7.10
PATCH 7/25 powerpc: Use encode avpn where we need only avpn values
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
In all these cases we are doing something similar to
HPTE_V_COMPARE(hpte_v, want_v) which ignores the HPTE_V_LARGE bit
With MPSS support we would need actual page size to set HPTE_V_LARGE bit and that won't be available in most of these cases. Since we are ignoring HPTE_V_LARGE bit, use the avpn value instead. There should not be any change in behaviour after this patch.
--- arch/powerpc/mm/hash_native_64.c | 8 ++++---- arch/powerpc/platforms/cell/beat_htab.c | 10 +++++----- arch/powerpc/platforms/ps3/htab.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-)
arch/powerpc/mm/hash_native_64.c +4 -4
@@ -252,7 +252,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,unsigned long hpte_v, want_v;int ret =0 ; - want_v =hpte_encode_v (vpn, psize, ssize); + want_v =hpte_encode_avpn (vpn, psize, ssize);DBG_LOW (" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)" , vpn, want_v &HPTE_V_AVPN , slot, newpp);
@@ -288,7 +288,7 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)unsigned long want_v, hpte_v; hash =hpt_hash (vpn, mmu_psize_defs[psize].shift, ssize); - want_v =hpte_encode_v (vpn, psize, ssize); + want_v =hpte_encode_avpn (vpn, psize, ssize);/* Bolted mappings are only ever in the primary group */ slot = (hash & htab_hash_mask) *HPTES_PER_GROUP ;
@@ -348,7 +348,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,DBG_LOW (" invalidate(vpn=%016lx, hash: %lx)\n" , vpn, slot); - want_v =hpte_encode_v (vpn, psize, ssize); + want_v =hpte_encode_avpn (vpn, psize, ssize);native_lock_hpte (hptep); hpte_v = hptep->v;
@@ -520,7 +520,7 @@ static void native_flush_hash_range(unsigned long number, int local) slot = (hash & htab_hash_mask) *HPTES_PER_GROUP ; slot += hidx & _PTEIDX_GROUP_IX; hptep = htab_address + slot; - want_v =hpte_encode_v (vpn, psize, ssize); + want_v =hpte_encode_avpn (vpn, psize, ssize);native_lock_hpte (hptep); hpte_v = hptep->v; if (!HPTE_V_COMPARE(hpte_v, want_v) ||
arch/powerpc/platforms/cell/beat_htab.c +5 -5
@@ -191,7 +191,7 @@ static long beat_lpar_hpte_updatepp(unsigned long slot,u64 dummy0, dummy1;unsigned long want_v; - want_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ); + want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M ); DBG_LOW(" update: " "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
@@ -228,7 +228,7 @@ static long beat_lpar_hpte_find(unsigned long vpn, int psize)unsigned long want_v, hpte_v; hash =hpt_hash (vpn, mmu_psize_defs[psize].shift,MMU_SEGSIZE_256M ); - want_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ); + want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M );for (j =0 ; j <2 ; j++) { slot = (hash & htab_hash_mask) *HPTES_PER_GROUP ;
@@ -283,7 +283,7 @@ static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,DBG_LOW (" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n" , slot, va, psize, local); - want_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ); + want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M );raw_spin_lock_irqsave (&beat_htab_lock, flags); dummy1 =beat_lpar_hpte_getword0 (slot);
@@ -372,7 +372,7 @@ static long beat_lpar_hpte_updatepp_v3(unsigned long slot,unsigned long want_v;unsigned long pss; - want_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ); + want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M ); pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc; DBG_LOW(" update: "
@@ -402,7 +402,7 @@ static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,DBG_LOW (" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n" , slot, vpn, psize, local); - want_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ); + want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M ); pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc; lpar_rc =beat_invalidate_htab_entry3 (0 , slot, want_v, pss);
arch/powerpc/platforms/ps3/htab.c +1 -1
@@ -115,7 +115,7 @@ static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp,unsigned long flags;long ret; - want_v =hpte_encode_v (vpn, psize, ssize); + want_v =hpte_encode_avpn (vpn, psize, ssize);spin_lock_irqsave (&ps3_htab_lock, flags);
Signature · 1 line
1.7.10
PATCH 8/25 powerpc: Decode the pte-lp-encoding bits correctly.
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We look at both the segment base page size and actual page size and store the pte-lp-encodings in an array per base page size.
We also update all relevant functions to take actual page size argument so that we can use the correct PTE LP encoding in HPTE. This should also get the basic Multiple Page Size per Segment (MPSS) support. This is needed to enable THP on ppc64.
--- arch/powerpc/include/asm/machdep.h | 3 +- arch/powerpc/include/asm/mmu-hash64.h | 33 +++++---- arch/powerpc/kvm/book3s_hv.c | 2 +- arch/powerpc/mm/hash_low_64.S | 18 +++-- arch/powerpc/mm/hash_native_64.c | 119 +++++++++++++++++++++--------- arch/powerpc/mm/hash_utils_64.c | 121 ++++++++++++++++++++----------- arch/powerpc/mm/hugetlbpage-hash64.c | 4 +- arch/powerpc/platforms/cell/beat_htab.c | 16 ++-- arch/powerpc/platforms/ps3/htab.c | 6 +- arch/powerpc/platforms/pseries/lpar.c | 6 +- 10 files changed, 214 insertions(+), 114 deletions(-)
arch/powerpc/include/asm/machdep.h +2 -1
@@ -50,7 +50,8 @@ struct machdep_calls {unsigned long prpn, unsigned long rflags, unsigned long vflags, - int psize, int ssize); + int psize, int apsize, + int ssize);long (*hpte_remove)(unsigned long hpte_group);void (*hpte_removebolted)(unsigned long ea,int psize,int ssize);
arch/powerpc/include/asm/mmu-hash64.h +19 -14
@@ -154,7 +154,7 @@ extern unsigned long htab_hash_mask;struct mmu_psize_def {unsigned int shift;/* number of bits */ -unsigned int penc;/* HPTE encoding */ +int penc[MMU_PAGE_COUNT ];/* HPTE encoding */ unsigned int tlbiel;/* tlbiel supported for that page size */ unsigned long avpnm;/* bits to mask out in AVPN in the HPTE */ unsigned long sllp;/* SLB L||LP (exact mask to use in slbmte) */
@@ -181,6 +181,13 @@ struct mmu_psize_def*/ #define VPN_SHIFT 12 +/* +* HPTE Large Page (LP) details +*/ +#define LP_SHIFT 12 +#define LP_BITS 8 +#define LP_MASK (i) ((0xFF >> (i)) << LP_SHIFT) +#ifndef __ASSEMBLY__static inline int segment_shift (int ssize)
@@ -237,14 +244,14 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,/* * This function sets the AVPN and L fields of the HPTE appropriately -* for the page size +* using the base page size and actual page size. */ -static inline unsigned long hpte_encode_v (unsigned long vpn, -int psize,int ssize) +static inline unsigned long hpte_encode_v (unsigned long vpn,int base_psize, +int actual_psize,int ssize) {unsigned long v; - v =hpte_encode_avpn (vpn, psize, ssize); -if (psize !=MMU_PAGE_4K ) + v =hpte_encode_avpn (vpn, base_psize, ssize); +if (actual_psize !=MMU_PAGE_4K ) v |=HPTE_V_LARGE ;return v; }
@@ -254,19 +261,17 @@ static inline unsigned long hpte_encode_v(unsigned long vpn,* for the page size. We assume the pa is already "clean" that is properly * aligned for the requested page size */ -static inline unsigned long hpte_encode_r (unsigned long pa,int psize) +static inline unsigned long hpte_encode_r (unsigned long pa,int base_psize, +int actual_psize) { -unsigned long r; -/* A 4K page needs no special encoding */ -if (psize ==MMU_PAGE_4K ) +if (actual_psize ==MMU_PAGE_4K )return pa &HPTE_R_RPN ;else { -unsigned int penc = mmu_psize_defs[psize].penc; -unsigned int shift = mmu_psize_defs[psize].shift; -return (pa & ~((1ul << shift) -1 )) | (penc <<12 ); +unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize]; +unsigned int shift = mmu_psize_defs[actual_psize].shift; +return (pa & ~((1ul << shift) -1 )) | (penc <<LP_SHIFT ); } -return r; }/*
arch/powerpc/kvm/book3s_hv.c +1 -1
@@ -1515,7 +1515,7 @@ static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, (*sps)->page_shift = def->shift; (*sps)->slb_enc = def->sllp; (*sps)->enc[0 ].page_shift = def->shift; - (*sps)->enc[0 ].pte_enc = def->penc; + (*sps)->enc[0 ].pte_enc = def->penc[linux_psize]; (*sps)++; }
arch/powerpc/mm/hash_low_64.S +12 -6
@@ -196,7 +196,8 @@ htab_insert_pte: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert1) bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0
@@ -219,7 +220,8 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert2) bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0
@@ -515,7 +517,8 @@ htab_special_pfn: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert1) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0
@@ -542,7 +545,8 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_4K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert2) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0
@@ -840,7 +844,8 @@ ht64_insert_pte: mr r4,r29 /* Retrieve vpn */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_64K - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_64K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(ht64_call_hpte_insert1) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0
@@ -863,7 +868,8 @@ _GLOBAL(ht64_call_hpte_insert1) mr r4,r29 /* Retrieve vpn */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_64K - ld r9,STK_PARAM(R9)(r1) /* segment size */ + li r9,MMU_PAGE_64K /* actual page size */ + ld r10,STK_PARAM(R9)(r1) /* segment size */ _GLOBAL(ht64_call_hpte_insert2) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0
arch/powerpc/mm/hash_native_64.c +85 -34
284 lines not on this page · raw message
arch/powerpc/mm/hash_utils_64.c +79 -42
183 lines not on this page · raw message
arch/powerpc/mm/hugetlbpage-hash64.c +2 -2
@@ -103,7 +103,7 @@ repeat:/* Insert into the hash table, primary slot */ slot = ppc_md.hpte_insert (hpte_group, vpn, pa, rflags,0 , - mmu_psize, ssize); + mmu_psize, mmu_psize, ssize);/* Primary is full, try the secondary */ if (unlikely (slot ==-1 )) {
@@ -111,7 +111,7 @@ repeat: HPTES_PER_GROUP) & ~0x7UL; slot = ppc_md.hpte_insert (hpte_group, vpn, pa, rflags,HPTE_V_SECONDARY , - mmu_psize, ssize); + mmu_psize, mmu_psize, ssize);if (slot ==-1 ) {if (mftb () &0x1 ) hpte_group = ((hash & htab_hash_mask) *
arch/powerpc/platforms/cell/beat_htab.c +8 -8
@@ -90,7 +90,7 @@ static inline unsigned int beat_read_mask(unsigned hpte_group)static long beat_lpar_hpte_insert (unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags, -int psize,int ssize) +int psize,int apsize,int ssize) {unsigned long lpar_rc;u64 hpte_v, hpte_r, slot;
@@ -103,9 +103,9 @@ static long beat_lpar_hpte_insert(unsigned long hpte_group,"rflags=%lx, vflags=%lx, psize=%d)\n" , hpte_group, va, pa, rflags, vflags, psize); - hpte_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ) | + hpte_v =hpte_encode_v (vpn, psize, apsize,MMU_SEGSIZE_256M ) | vflags |HPTE_V_VALID ; - hpte_r =hpte_encode_r (pa, psize) | rflags; + hpte_r =hpte_encode_r (pa, psize, apsize) | rflags;if (!(vflags &HPTE_V_BOLTED ))DBG_LOW (" hpte_v=%016lx, hpte_r=%016lx\n" , hpte_v, hpte_r);
@@ -314,7 +314,7 @@ void __init hpte_init_beat(void)static long beat_lpar_hpte_insert_v3 (unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags, -int psize,int ssize) +int psize,int apsize,int ssize) {unsigned long lpar_rc;u64 hpte_v, hpte_r, slot;
@@ -327,9 +327,9 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,"rflags=%lx, vflags=%lx, psize=%d)\n" , hpte_group, vpn, pa, rflags, vflags, psize); - hpte_v =hpte_encode_v (vpn, psize,MMU_SEGSIZE_256M ) | + hpte_v =hpte_encode_v (vpn, psize, apsize,MMU_SEGSIZE_256M ) | vflags |HPTE_V_VALID ; - hpte_r =hpte_encode_r (pa, psize) | rflags; + hpte_r =hpte_encode_r (pa, psize, apsize) | rflags;if (!(vflags &HPTE_V_BOLTED ))DBG_LOW (" hpte_v=%016lx, hpte_r=%016lx\n" , hpte_v, hpte_r);
@@ -373,7 +373,7 @@ static long beat_lpar_hpte_updatepp_v3(unsigned long slot,unsigned long pss; want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M ); - pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc; + pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc[psize]; DBG_LOW(" update: " "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
@@ -403,7 +403,7 @@ static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,DBG_LOW (" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n" , slot, vpn, psize, local); want_v =hpte_encode_avpn (vpn, psize,MMU_SEGSIZE_256M ); - pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc; + pss = (psize ==MMU_PAGE_4K ) ?-1UL : mmu_psize_defs[psize].penc[psize]; lpar_rc =beat_invalidate_htab_entry3 (0 , slot, want_v, pss);
arch/powerpc/platforms/ps3/htab.c +3 -3
@@ -45,7 +45,7 @@ static DEFINE_SPINLOCK(ps3_htab_lock);static long ps3_hpte_insert (unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags, -int psize,int ssize) +int psize,int apsize,int ssize) {int result;u64 hpte_v, hpte_r;
@@ -61,8 +61,8 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn,*/ vflags &= ~HPTE_V_SECONDARY ; - hpte_v =hpte_encode_v (vpn, psize, ssize) | vflags |HPTE_V_VALID ; - hpte_r =hpte_encode_r (ps3_mm_phys_to_lpar (pa), psize) | rflags; + hpte_v =hpte_encode_v (vpn, psize, apsize, ssize) | vflags |HPTE_V_VALID ; + hpte_r =hpte_encode_r (ps3_mm_phys_to_lpar (pa), psize, apsize) | rflags;spin_lock_irqsave (&ps3_htab_lock, flags);
arch/powerpc/platforms/pseries/lpar.c +3 -3
@@ -109,7 +109,7 @@ void vpa_init(int cpu)static long pSeries_lpar_hpte_insert (unsigned long hpte_group,unsigned long vpn,unsigned long pa,unsigned long rflags,unsigned long vflags, -int psize,int ssize) +int psize,int apsize,int ssize) {unsigned long lpar_rc;unsigned long flags;
@@ -121,8 +121,8 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,"pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n" , hpte_group, vpn, pa, rflags, vflags, psize); - hpte_v =hpte_encode_v (vpn, psize, ssize) | vflags |HPTE_V_VALID ; - hpte_r =hpte_encode_r (pa, psize) | rflags; + hpte_v =hpte_encode_v (vpn, psize, apsize, ssize) | vflags |HPTE_V_VALID ; + hpte_r =hpte_encode_r (pa, psize, apsize) | rflags;if (!(vflags &HPTE_V_BOLTED ))pr_devel (" hpte_v=%016lx, hpte_r=%016lx\n" , hpte_v, hpte_r);
Signature · 1 line
1.7.10
PATCH 9/25 powerpc: Fix hpte_decode to use the correct decoding for page sizes
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
As per ISA doc, we encode base and actual page size in the LP bits of PTE. The number of bit used to encode the page sizes depend on actual page size. ISA doc lists this as
PTE LP actual page size
rrrr rrrz ≥8KB
rrrr rrzz ≥16KB
rrrr rzzz ≥32KB
rrrr zzzz ≥64KB
rrrz zzzz ≥128KB
rrzz zzzz ≥256KB
rzzz zzzz ≥512KB
zzzz zzzz ≥1MB
ISA doc also says
"The values of the “z” bits used to specify each size, along with all possible values of “r” bits in the LP field, must result in LP values distinct from other LP values for other sizes."
based on the above update hpte_decode to use the correct decoding for LP bits.
--- arch/powerpc/mm/hash_native_64.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-)
arch/powerpc/mm/hash_native_64.c +24 -14
@@ -415,41 +415,51 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, int *psize, int *apsize, int *ssize, unsigned long *vpn) { unsigned long avpn, pteg, vpi; -unsigned long hpte_r = hpte->r;unsigned long hpte_v = hpte->v;unsigned long vsid, seg_off; -int i, size, a_size, shift, penc; +int size, a_size, shift, mask; +/* Look at the 8 bit LP value */ +unsigned int lp = (hpte->r >>LP_SHIFT ) & ((1 <<LP_BITS ) -1 );if (!(hpte_v &HPTE_V_LARGE )) { size =MMU_PAGE_4K ; a_size =MMU_PAGE_4K ; }else { -for (i =0 ; i <LP_BITS ; i++) { -if ((hpte_r &LP_MASK (i+1 )) ==LP_MASK (i+1 )) -break ; - } - penc =LP_MASK (i+1 ) >>LP_SHIFT ;for (size =0 ; size <MMU_PAGE_COUNT ; size++) {/* valid entries have a shift value */ if (!mmu_psize_defs[size].shift)continue ; -for (a_size =0 ; a_size <MMU_PAGE_COUNT ; a_size++) { - -/* 4K pages are not represented by LP */ -if (a_size ==MMU_PAGE_4K ) -continue ; +/* start from 1 ignoring MMU_PAGE_4K */ +for (a_size =1 ; a_size <MMU_PAGE_COUNT ; a_size++) {/* valid entries have a shift value */ if (!mmu_psize_defs[a_size].shift)continue ; -if (penc == mmu_psize_defs[size].penc[a_size]) +/* invalid penc */ +if (mmu_psize_defs[size].penc[a_size] ==-1 ) +continue ; +/* +* encoding bits per actual page size +* PTE LP actual page size +* rrrr rrrz ≥8KB +* rrrr rrzz ≥16KB +* rrrr rzzz ≥32KB +* rrrr zzzz ≥64KB +* ....... +*/ + shift = mmu_psize_defs[a_size].shift -LP_SHIFT ; +if (shift >LP_BITS ) + shift =LP_BITS ; + mask = (1 << shift) -1 ; +if ((lp & mask) == + mmu_psize_defs[size].penc[a_size]) { goto out; + } } } } - out:/* This works for all page sizes, and for 256M and 1T segments */ *ssize = hpte_v >>HPTE_V_SSIZE_SHIFT ;
Signature · 1 line
1.7.10
PATCH 10/25 powerpc: Return all the valid pte ecndoing in KVM_PPC_GET_SMMU_INFO ioctl
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
--- arch/powerpc/kvm/book3s_hv.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)
arch/powerpc/kvm/book3s_hv.c +12 -2
@@ -1508,14 +1508,24 @@ long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, struct kvm_allocate_rma *ret)static void kvmppc_add_seg_page_size (struct kvm_ppc_one_seg_page_size **sps,int linux_psize) { +int i, index =0 ;struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];if (!def->shift)return ; (*sps)->page_shift = def->shift; (*sps)->slb_enc = def->sllp; - (*sps)->enc[0 ].page_shift = def->shift; - (*sps)->enc[0 ].pte_enc = def->penc[linux_psize]; +for (i =0 ; i <MMU_PAGE_COUNT ; i++) { +if (def->penc[i] !=-1 ) { +if (index >=KVM_PPC_PAGE_SIZES_MAX_SZ ) { +WARN_ON (1 ); +break ; + } + (*sps)->enc[index].page_shift = mmu_psize_defs[i].shift; + (*sps)->enc[index].pte_enc = def->penc[i]; + index++; + } + } (*sps)++; }
Signature · 1 line
1.7.10
PATCH 11/25 powerpc: Update tlbie/tlbiel as per ISA doc
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
This make sure we handle multiple page size segment correctly.
--- arch/powerpc/mm/hash_native_64.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)
arch/powerpc/mm/hash_native_64.c +30 -2
@@ -61,7 +61,10 @@ static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)switch (psize) {case MMU_PAGE_4K : +/* clear out bits after (52) [0....52.....63] */ + va &= ~((1ul << (64 -52 )) -1 ); va |= ssize <<8 ; + va |= mmu_psize_defs[apsize].sllp <<6 ; asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) : "memory");
@@ -69,9 +72,20 @@ static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)default :/* We need 14 to 14 + i bits of va */ penc = mmu_psize_defs[psize].penc[apsize]; - va &= ~((1ul << mmu_psize_defs[psize].shift) -1 ); +/* clear out bits after (44) [0....44.....63] */ + va &= ~((1ul << (64 -44 )) -1 ); va |= penc <<12 ; va |= ssize <<8 ; +/* Add AVAL part */ +if (psize != apsize) { +/* +* MPSS, 64K base page size and 16MB parge page size +* We don't need all the bits, but this seems to work. +* vpn cover upto 65 bits of va. (0...65) and we need +* 56..62 bits of va. +*/ + va |= ((vpn >>2 ) &0xfe ); + } va |=1 ;/* L */ asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
@@ -96,16 +110,30 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)switch (psize) {case MMU_PAGE_4K : +/* clear out bits after(52) [0....52.....63] */ + va &= ~((1ul << (64 -52 )) -1 ); va |= ssize <<8 ; + va |= mmu_psize_defs[apsize].sllp <<6 ; asmvolatile (".long 0x7c000224 | (%0 << 11) | (0 << 21)" : :"r" (va) :"memory" );break ;default :/* We need 14 to 14 + i bits of va */ penc = mmu_psize_defs[psize].penc[apsize]; - va &= ~((1ul << mmu_psize_defs[psize].shift) -1 ); +/* clear out bits after(44) [0....44.....63] */ + va &= ~((1ul << (64 -44 )) -1 ); va |= penc <<12 ; va |= ssize <<8 ; +/* Add AVAL part */ +if (psize != apsize) { +/* +* MPSS, 64K base page size and 16MB parge page size +* We don't need all the bits, but this seems to work. +* vpn cover upto 65 bits of va. (0...65) and we need +* 56..62 bits of va. +*/ + va |= ((vpn >>2 ) &0xfe ); + } va |=1 ;/* L */ asmvolatile (".long 0x7c000224 | (%0 << 11) | (1 << 21)" : :"r" (va) :"memory" );
Signature · 1 line
1.7.10
PATCH 12/25 powerpc: print both base and actual page size on hash failure
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
--- arch/powerpc/include/asm/mmu-hash64.h | 3 ++- arch/powerpc/mm/hash_utils_64.c | 12 +++++++----- arch/powerpc/mm/hugetlbpage-hash64.c | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-)
arch/powerpc/include/asm/mmu-hash64.h +2 -1
@@ -324,7 +324,8 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, unsigned int shift, unsigned int mmu_psize);extern void hash_failure_debug (unsigned long ea,unsigned long access,unsigned long vsid,unsigned long trap, -int ssize,int psize,unsigned long pte); +int ssize,int psize,int lpsize, +unsigned long pte);extern int htab_bolt_mapping (unsigned long vstart,unsigned long vend,unsigned long pstart,unsigned long prot,int psize,int ssize);
arch/powerpc/mm/hash_utils_64.c +7 -5
@@ -933,14 +933,14 @@ static inline int subpage_protection(struct mm_struct *mm, unsigned long ea)void hash_failure_debug (unsigned long ea,unsigned long access,unsigned long vsid,unsigned long trap, -int ssize,int psize,unsigned long pte) +int ssize,int psize,int lpsize,unsigned long pte) {if (!printk_ratelimit ())return ;pr_info ("mm: Hashing failure ! EA=0x%lx access=0x%lx current=%s\n" , ea, access, current->comm); -pr_info (" trap=0x%lx vsid=0x%lx ssize=%d psize=%d pte=0x%lx\n" , - trap, vsid, ssize, psize, pte); +pr_info (" trap=0x%lx vsid=0x%lx ssize=%d base psize=%d psize %d pte=0x%lx\n" , + trap, vsid, ssize, psize, lpsize, pte); }/* Result code is:
@@ -1113,7 +1113,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)*/ if (rc ==-1 )hash_failure_debug (ea, access, vsid, trap, ssize, psize, -pte_val (*ptep)); + psize,pte_val (*ptep));#ifndef CONFIG_PPC_64K_PAGES DBG_LOW (" o-pte: %016lx\n" ,pte_val (*ptep));#else
@@ -1191,7 +1191,9 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,*/ if (rc ==-1 )hash_failure_debug (ea, access, vsid, trap, ssize, - mm->context.user_psize,pte_val (*ptep)); + mm->context.user_psize, + mm->context.user_psize, +pte_val (*ptep));local_irq_restore (flags); }
arch/powerpc/mm/hugetlbpage-hash64.c +1 -1
@@ -129,7 +129,7 @@ repeat:if (unlikely (slot ==-2 )) { *ptep =__pte (old_pte);hash_failure_debug (ea, access, vsid, trap, ssize, - mmu_psize, old_pte); + mmu_psize, mmu_psize, old_pte);return -1 ; }
Signature · 1 line
1.7.10
PATCH 13/25 powerpc: Print page size info during boot
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
This gives hint about different base and actual page size combination supported by the platform.
--- arch/powerpc/mm/hash_utils_64.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/powerpc/mm/hash_utils_64.c +5 -5
@@ -315,7 +315,7 @@ static int __init htab_dt_scan_page_sizes(unsigned long node, prop = (u32 *)of_get_flat_dt_prop (node,"ibm,segment-page-sizes" , &size);if (prop !=NULL ) { -DBG ("Page sizes from device-tree:\n" ); +pr_info ("Page sizes from device-tree:\n" ); size /=4 ; cur_cpu_spec->mmu_features &= ~(MMU_FTR_16M_PAGE );while (size >0 ) {
@@ -369,10 +369,10 @@ static int __init htab_dt_scan_page_sizes(unsigned long node, "shift=%d\n", base_shift, shift); def->penc[idx] = penc; -DBG (" %d: shift=%02x, sllp=%04lx, " -"avpnm=%08lx, tlbiel=%d, penc=%d\n" , - idx, shift, def->sllp, def->avpnm, - def->tlbiel, def->penc[idx]); +pr_info ("base_shift=%d: shift=%d, sllp=0x%04lx," +" avpnm=0x%08lx, tlbiel=%d, penc=%d\n" , + base_shift, shift, def->sllp, + def->avpnm, def->tlbiel, def->penc[idx]); } }return 1 ;
Signature · 1 line
1.7.10
PATCH 14/25 mm/THP: HPAGE_SHIFT is not a #define on some arch
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
On archs like powerpc that support different hugepage sizes, HPAGE_SHIFT and other derived values like HPAGE_PMD_ORDER are not constants. So move that to hugepage_init
--- include/linux/huge_mm.h | 3 --- mm/huge_memory.c | 9 ++++++--- 2 files changed, 6 insertions(+), 6 deletions(-)
include/linux/huge_mm.h +0 -3
@@ -119,9 +119,6 @@ extern void __split_huge_page_pmd(struct vm_area_struct *vma, }while (0 ) extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,pmd_t *pmd); -#if HPAGE_PMD_ORDER >MAX_ORDER -#error "hugepages can't be allocated by the buddy allocator" -#endif extern int hugepage_madvise (struct vm_area_struct *vma,unsigned long *vm_flags,int advice); extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
mm/huge_memory.c +6 -3
@@ -44,7 +44,7 @@ unsigned long transparent_hugepage_flags __read_mostly = (1 <<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG );/* default scan 8*512 pte (or vmas) every 30 second */ -static unsigned int khugepaged_pages_to_scan__read_mostly =HPAGE_PMD_NR *8 ; +static unsigned int khugepaged_pages_to_scan__read_mostly ;static unsigned int khugepaged_pages_collapsed;static unsigned int khugepaged_full_scans;static unsigned int khugepaged_scan_sleep_millisecs__read_mostly =10000 ;
@@ -59,7 +59,7 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);* it would have happened if the vma was large enough during page * fault. */ -static unsigned int khugepaged_max_ptes_none__read_mostly =HPAGE_PMD_NR -1 ; +static unsigned int khugepaged_max_ptes_none__read_mostly ;static int khugepaged (void *none);static int mm_slots_hash_init (void );
@@ -621,11 +621,14 @@ static int __init hugepage_init(void)int err;struct kobject *hugepage_kobj; -if (!has_transparent_hugepage ()) { +if (!has_transparent_hugepage () || (HPAGE_PMD_ORDER >MAX_ORDER )) { transparent_hugepage_flags =0 ;return -EINVAL ; } + khugepaged_pages_to_scan =HPAGE_PMD_NR *8 ; + khugepaged_max_ptes_none =HPAGE_PMD_NR -1 ; + err =hugepage_init_sysfs (&hugepage_kobj);if (err)return err;
Signature · 1 line
1.7.10
PATCH 15/25 mm/THP: Add pmd args to pgtable deposit and withdraw APIs
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
This will be later used by powerpc THP support. In powerpc we want to use pgtable for storing the hash index values. So instead of adding them to mm_context list, we would like to store them in the second half of pmd
--- arch/s390/include/asm/pgtable.h | 5 +++-- arch/s390/mm/pgtable.c | 5 +++-- arch/sparc/include/asm/pgtable_64.h | 5 +++-- arch/sparc/mm/tlb.c | 5 +++-- include/asm-generic/pgtable.h | 5 +++-- mm/huge_memory.c | 18 +++++++++--------- mm/pgtable-generic.c | 5 +++-- 7 files changed, 27 insertions(+), 21 deletions(-)
arch/s390/include/asm/pgtable.h +3 -2
@@ -1232,10 +1232,11 @@ static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW)#define __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pgtable_t pgtable); +extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pmd_t *pmdp, +pgtable_t pgtable);#define __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm); +extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp);static inline int pmd_trans_splitting (pmd_t pmd) {
arch/s390/mm/pgtable.c +3 -2
@@ -920,7 +920,8 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, } } -void pgtable_trans_huge_deposit (struct mm_struct *mm,pgtable_t pgtable) +void pgtable_trans_huge_deposit (struct mm_struct *mm,pmd_t *pmdp, +pgtable_t pgtable) {struct list_head *lh = (struct list_head *) pgtable;
@@ -934,7 +935,7 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) mm->pmd_huge_pte = pgtable; } -pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm) +pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp) {struct list_head *lh;pgtable_t pgtable;
arch/sparc/include/asm/pgtable_64.h +3 -2
@@ -853,10 +853,11 @@ extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd);#define __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pgtable_t pgtable); +extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pmd_t *pmdp, +pgtable_t pgtable);#define __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm); +extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp);#endif /* Encode and de-code a swap entry */
arch/sparc/mm/tlb.c +3 -2
@@ -150,7 +150,8 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, } } -void pgtable_trans_huge_deposit (struct mm_struct *mm,pgtable_t pgtable) +void pgtable_trans_huge_deposit (struct mm_struct *mm,pmd_t *pmdp, +pgtable_t pgtable) {struct list_head *lh = (struct list_head *) pgtable;
@@ -164,7 +165,7 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) mm->pmd_huge_pte = pgtable; } -pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm) +pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp) {struct list_head *lh;pgtable_t pgtable;
include/asm-generic/pgtable.h +3 -2
@@ -163,11 +163,12 @@ extern void pmdp_splitting_flush(struct vm_area_struct *vma,#endif #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pgtable_t pgtable); +extern void pgtable_trans_huge_deposit (struct mm_struct *mm,pmd_t *pmdp, +pgtable_t pgtable);#endif #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm); +extern pgtable_t pgtable_trans_huge_withdraw (struct mm_struct *mm,pmd_t *pmdp);#endif #ifndef __HAVE_ARCH_PMDP_INVALIDATE
mm/huge_memory.c +9 -9
@@ -742,7 +742,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,*/ page_add_new_anon_rmap (page, vma, haddr);set_pmd_at (mm, haddr, pmd, entry); -pgtable_trans_huge_deposit (mm, pgtable); +pgtable_trans_huge_deposit (mm, pmd, pgtable);add_mm_counter (mm,MM_ANONPAGES ,HPAGE_PMD_NR ); mm->nr_ptes++;spin_unlock (&mm->page_table_lock);
@@ -784,7 +784,7 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, entry =pmd_wrprotect (entry); entry =pmd_mkhuge (entry);set_pmd_at (mm, haddr, pmd, entry); -pgtable_trans_huge_deposit (mm, pgtable); +pgtable_trans_huge_deposit (mm, pmd, pgtable); mm->nr_ptes++;return true; }
@@ -929,7 +929,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,pmdp_set_wrprotect (src_mm, addr, src_pmd); pmd =pmd_mkold (pmd_wrprotect (pmd));set_pmd_at (dst_mm, addr, dst_pmd, pmd); -pgtable_trans_huge_deposit (dst_mm, pgtable); +pgtable_trans_huge_deposit (dst_mm, dst_pmd, pgtable); dst_mm->nr_ptes++; ret =0 ;
@@ -999,7 +999,7 @@ static int do_huge_pmd_wp_zero_page_fallback(struct mm_struct *mm,pmdp_clear_flush (vma, haddr, pmd);/* leave pmd empty until pte is filled */ - pgtable =pgtable_trans_huge_withdraw (mm); + pgtable =pgtable_trans_huge_withdraw (mm, pmd);pmd_populate (mm, &_pmd, pgtable);for (i =0 ; i <HPAGE_PMD_NR ; i++, haddr +=PAGE_SIZE ) {
@@ -1094,10 +1094,10 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, goto out_free_pages;VM_BUG_ON (!PageHead (page)); + pgtable =pgtable_trans_huge_withdraw (mm, pmd);pmdp_clear_flush (vma, haddr, pmd);/* leave pmd empty until pte is filled */ - pgtable =pgtable_trans_huge_withdraw (mm);pmd_populate (mm, &_pmd, pgtable);for (i =0 ; i <HPAGE_PMD_NR ; i++, haddr +=PAGE_SIZE ) {
@@ -1380,7 +1380,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,struct page *page;pgtable_t pgtable;pmd_t orig_pmd; - pgtable =pgtable_trans_huge_withdraw (tlb->mm); + pgtable =pgtable_trans_huge_withdraw (tlb->mm, pmd); orig_pmd =pmdp_get_and_clear (tlb->mm, addr, pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr);if (is_huge_zero_pmd (orig_pmd)) {
@@ -1712,7 +1712,7 @@ static int __split_huge_page_map(struct page *page, pmd =page_check_address_pmd (page, mm, address,PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG );if (pmd) { - pgtable =pgtable_trans_huge_withdraw (mm); + pgtable =pgtable_trans_huge_withdraw (mm, pmd);pmd_populate (mm, &_pmd, pgtable); haddr = address;
@@ -2400,7 +2400,7 @@ static void collapse_huge_page(struct mm_struct *mm,page_add_new_anon_rmap (new_page, vma, address);set_pmd_at (mm, address, pmd, _pmd);update_mmu_cache_pmd (vma, address, pmd); -pgtable_trans_huge_deposit (mm, pgtable); +pgtable_trans_huge_deposit (mm, pmd, pgtable);spin_unlock (&mm->page_table_lock); *hpage =NULL ;
@@ -2706,7 +2706,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,pmdp_clear_flush (vma, haddr, pmd);/* leave pmd empty until pte is filled */ - pgtable =pgtable_trans_huge_withdraw (mm); + pgtable =pgtable_trans_huge_withdraw (mm, pmd);pmd_populate (mm, &_pmd, pgtable);for (i =0 ; i <HPAGE_PMD_NR ; i++, haddr +=PAGE_SIZE ) {
mm/pgtable-generic.c +3 -2
@@ -124,7 +124,8 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT #ifdef CONFIG_TRANSPARENT_HUGEPAGE -void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable) { assert_spin_locked(&mm->page_table_lock);
@@ -141,7 +142,7 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* no "address" argument so destroys page coloring of some arch */ -pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm) +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable;
Signature · 1 line
1.7.10
PATCH 16/25 mm/THP: withdraw the pgtable after pmdp related operations
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
For architectures like ppc64 we look at deposited pgtable when calling pmdp_get_and_clear. So do the pgtable_trans_huge_withdraw after finishing pmdp related operations.
--- mm/huge_memory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
mm/huge_memory.c +2 -1
@@ -1380,9 +1380,10 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,struct page *page;pgtable_t pgtable;pmd_t orig_pmd; - pgtable =pgtable_trans_huge_withdraw (tlb->mm, pmd); + orig_pmd =pmdp_get_and_clear (tlb->mm, addr, pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr); + pgtable =pgtable_trans_huge_withdraw (tlb->mm, pmd);if (is_huge_zero_pmd (orig_pmd)) { tlb->mm->nr_ptes--;spin_unlock (&tlb->mm->page_table_lock);
Signature · 1 line
1.7.10
PATCH 17/25 powerpc/THP: Implement transparent hugepages for ppc64
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We now have pmd entries covering to 16MB range. To implement THP on powerpc, we double the size of PMD. The second half is used to deposit the pgtable (PTE page). We also use the depoisted PTE page for tracking the HPTE information. The information include [ secondary group | 3 bit hidx | valid ]. We use one byte per each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and with 4K HPTE we need 4096 entries. Both will fit in a 4K PTE page.
--- arch/powerpc/include/asm/page.h | 2 +- arch/powerpc/include/asm/pgtable-ppc64-64k.h | 3 +- arch/powerpc/include/asm/pgtable-ppc64.h | 2 +- arch/powerpc/include/asm/pgtable.h | 240 ++++++++++++++++++++ arch/powerpc/mm/pgtable.c | 314 ++++++++++++++++++++++++++ arch/powerpc/mm/pgtable_64.c | 13 ++ arch/powerpc/platforms/Kconfig.cputype | 1 + 7 files changed, 572 insertions(+), 3 deletions(-)
arch/powerpc/include/asm/page.h +1 -1
@@ -40,7 +40,7 @@#ifdef CONFIG_HUGETLB_PAGE extern unsigned int HPAGE_SHIFT ;#else -#define HPAGE_SHIFT PAGE_SHIFT +#define HPAGE_SHIFT PMD_SHIFT#endif #define HPAGE_SIZE ((1UL) << HPAGE_SHIFT)#define HPAGE_MASK (~(HPAGE_SIZE - 1))
arch/powerpc/include/asm/pgtable-ppc64-64k.h +2 -1
@@ -33,7 +33,8 @@#define PGDIR_MASK (~(PGDIR_SIZE-1))/* Bits to mask out from a PMD to get to the PTE page */ -#define PMD_MASKED_BITS 0x1ff +/* PMDs point to PTE table fragments which are 4K aligned. */ +#define PMD_MASKED_BITS 0xfff/* Bits to mask out from a PGD/PUD to get to the PMD page */ #define PUD_MASKED_BITS 0x1ff
arch/powerpc/include/asm/pgtable-ppc64.h +1 -1
@@ -150,7 +150,7 @@#define pmd_present (pmd) (pmd_val(pmd) != 0)#define pmd_clear (pmdp) (pmd_val(*(pmdp)) = 0)#define pmd_page_vaddr (pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) -#define pmd_page (pmd) virt_to_page(pmd_page_vaddr(pmd)) +extern struct page *pmd_page (pmd_t pmd);#define pud_set (pudp, pudval) (pud_val(*(pudp)) = (pudval))#define pud_none (pud) (!pud_val(pud))
arch/powerpc/include/asm/pgtable.h +240 -0
247 lines not on this page · raw message
arch/powerpc/mm/pgtable.c +314 -0
323 lines not on this page · raw message
arch/powerpc/mm/pgtable_64.c +13 -0
@@ -338,6 +338,19 @@ EXPORT_SYMBOL(iounmap);EXPORT_SYMBOL (__iounmap);EXPORT_SYMBOL (__iounmap_at); +/* +* For hugepage we have pfn in the pmd, we use PMD_HUGE_RPN_SHIFT bits for flags +* For PTE page, we have a PTE_FRAG_SIZE (4K) aligned virtual address. +*/ +struct page *pmd_page (pmd_t pmd) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +if (pmd_val (pmd) &PMD_ISHUGE ) +return pfn_to_page (pmd_pfn (pmd)); +#endif +return virt_to_page (pmd_page_vaddr (pmd)); +} +#ifdef CONFIG_PPC_64K_PAGES /* * we support 16 fragments per PTE page. This is limited by how many
arch/powerpc/platforms/Kconfig.cputype +1 -0
@@ -71,6 +71,7 @@ config PPC_BOOK3S_64 select PPC_FPU select PPC_HAVE_PMU_SUPPORT select SYS_SUPPORTS_HUGETLBFS + select HAVE_ARCH_TRANSPARENT_HUGEPAGE if PPC_64K_PAGES config PPC_BOOK3E_64 bool "Embedded processors"
Signature · 1 line
1.7.10
PATCH 18/25 powerpc/THP: Double the PMD table size for THP
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
THP code does PTE page allocation along with large page request and deposit them for later use. This is to ensure that we won't have any failures when we split hugepages to regular pages.
On powerpc we want to use the deposited PTE page for storing hash pte slot and secondary bit information for the HPTEs. We use the second half of the pmd table to save the deposted PTE page.
--- arch/powerpc/include/asm/pgalloc-64.h | 6 +++--- arch/powerpc/include/asm/pgtable-ppc64.h | 6 +++++- arch/powerpc/mm/init_64.c | 9 ++++++--- 3 files changed, 14 insertions(+), 7 deletions(-)
arch/powerpc/include/asm/pgalloc-64.h +3 -3
@@ -208,17 +208,17 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,static inline pmd_t *pmd_alloc_one (struct mm_struct *mm,unsigned long addr) { -return kmem_cache_alloc (PGT_CACHE (PMD_INDEX_SIZE ), +return kmem_cache_alloc (PGT_CACHE (PMD_CACHE_INDEX ),GFP_KERNEL |__GFP_REPEAT); }static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) { -kmem_cache_free (PGT_CACHE (PMD_INDEX_SIZE ), pmd); +kmem_cache_free (PGT_CACHE (PMD_CACHE_INDEX ), pmd); }#define __pmd_free_tlb (tlb, pmd, addr) \ - pgtable_free_tlb(tlb, pmd, PMD_INDEX_SIZE) + pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX)#ifndef CONFIG_PPC_64K_PAGES #define __pud_free_tlb (tlb, pud, addr) \ pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE)
arch/powerpc/include/asm/pgtable-ppc64.h +5 -1
@@ -20,7 +20,11 @@ PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) - +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1) +#else +#define PMD_CACHE_INDEX PMD_INDEX_SIZE +#endif /* * Define the address range of the kernel non-linear virtual area */
arch/powerpc/mm/init_64.c +6 -3
@@ -88,7 +88,11 @@ static void pgd_ctor(void *addr)static void pmd_ctor (void *addr) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +memset (addr,0 ,PMD_TABLE_SIZE *2 ); +#else memset (addr,0 ,PMD_TABLE_SIZE ); +#endif }struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE ];
@@ -138,10 +142,9 @@ void pgtable_cache_add(unsigned shift, void (*ctor)(void *))void pgtable_cache_init (void ) {pgtable_cache_add (PGD_INDEX_SIZE , pgd_ctor); -pgtable_cache_add (PMD_INDEX_SIZE , pmd_ctor); -if (!PGT_CACHE (PGD_INDEX_SIZE ) || !PGT_CACHE (PMD_INDEX_SIZE )) +pgtable_cache_add (PMD_CACHE_INDEX , pmd_ctor); +if (!PGT_CACHE (PGD_INDEX_SIZE ) || !PGT_CACHE (PMD_CACHE_INDEX ))panic ("Couldn't allocate pgtable caches" ); -/* In all current configs, when the PUD index exists it's the * same size as either the pgd or pmd index. Verify that the * initialization above has also created a PUD cache. This
Signature · 1 line
1.7.10
PATCH 19/25 powerpc/THP: Differentiate THP PMD entries from HUGETLB PMD entries
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
HUGETLB clear the top bit of PMD entries and use that to indicate a HUGETLB page directory. Since we store pfns in PMDs for THP, we would have the top bit cleared by default. Add the top bit mask for THP PMD entries and clear that when we are looking for pmd_pfn.
--- arch/powerpc/include/asm/pgtable.h | 16 +++++++++++++--- arch/powerpc/mm/pgtable.c | 5 ++++- arch/powerpc/mm/pgtable_64.c | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-)
arch/powerpc/include/asm/pgtable.h +13 -3
@@ -31,7 +31,7 @@ struct mm_struct;#define PMD_HUGE_SPLITTING 0x008#define PMD_HUGE_SAO 0x010/* strong Access order */ #define PMD_HUGE_HASHPTE 0x020 -#define PMD_ISHUGE 0x040 +#define _PMD_ISHUGE 0x040#define PMD_HUGE_DIRTY 0x080/* C: page changed */ #define PMD_HUGE_ACCESSED 0x100/* R: page referenced */ #define PMD_HUGE_RW 0x200/* software: user write access allowed */
@@ -44,6 +44,14 @@ struct mm_struct;#define PMD_HUGE_RPN_SHIFT PTE_RPN_SHIFT#define HUGE_PAGE_SIZE (ASM_CONST(1) << 24)#define HUGE_PAGE_MASK (~(HUGE_PAGE_SIZE - 1)) +/* +* HugeTLB looks at the top bit of the Linux page table entries to +* decide whether it is a huge page directory or not. Mark HUGE +* PMD to differentiate +*/ +#define PMD_HUGE_NOT_HUGETLB (ASM_CONST(1) << 63) +#define PMD_ISHUGE (_PMD_ISHUGE | PMD_HUGE_NOT_HUGETLB) +#define PMD_HUGE_PROTBITS (0xfff | PMD_HUGE_NOT_HUGETLB)#ifndef __ASSEMBLY__ extern void hpte_need_hugepage_flush(struct mm_struct *mm, unsigned long addr,
@@ -70,8 +78,9 @@ static inline int pmd_trans_splitting(pmd_t pmd)static inline int pmd_trans_huge (pmd_t pmd) { -return pmd_val (pmd) &PMD_ISHUGE ; +return ((pmd_val (pmd) &PMD_ISHUGE ) ==PMD_ISHUGE ); } +/* We will enable it in the last patch */ #define has_transparent_hugepage () 0#else
@@ -84,7 +93,8 @@ static inline unsigned long pmd_pfn(pmd_t pmd)/* * Only called for hugepage pmd */ -return pmd_val (pmd) >>PMD_HUGE_RPN_SHIFT ; +unsigned long val =pmd_val (pmd) & ~PMD_HUGE_PROTBITS ; +return val >>PMD_HUGE_RPN_SHIFT ; }static inline int pmd_young (pmd_t pmd)
arch/powerpc/mm/pgtable.c +4 -1
@@ -517,7 +517,10 @@ static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)pmd_t pfn_pmd (unsigned long pfn,pgprot_t pgprot) {pmd_t pmd; - +/* +* We cannot support that many PFNs +*/ +VM_BUG_ON (pfn &PMD_HUGE_NOT_HUGETLB );pmd_val (pmd) = pfn <<PMD_HUGE_RPN_SHIFT ;pmd_val (pmd) |=PMD_ISHUGE ; pmd =pmd_set_protbits (pmd, pgprot);
arch/powerpc/mm/pgtable_64.c +1 -1
@@ -345,7 +345,7 @@ EXPORT_SYMBOL(__iounmap_at);struct page *pmd_page (pmd_t pmd) {#ifdef CONFIG_TRANSPARENT_HUGEPAGE -if (pmd_val (pmd) &PMD_ISHUGE ) +if ((pmd_val (pmd) &PMD_ISHUGE ) ==PMD_ISHUGE )return pfn_to_page (pmd_pfn (pmd));#endif return virt_to_page (pmd_page_vaddr (pmd));
Signature · 1 line
1.7.10
PATCH 20/25 powerpc/THP: Add code to handle HPTE faults for large pages
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We now have pmd entries covering to 16MB range. To implement THP on powerpc, we double the size of PMD. The second half is used to deposit the pgtable (PTE page). We also use the depoisted PTE page for tracking the HPTE information. The information include [ secondary group | 3 bit hidx | valid ]. We use one byte per each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and with 4K HPTE we need 4096 entries. Both will fit in a 4K PTE page.
--- arch/powerpc/include/asm/mmu-hash64.h | 5 + arch/powerpc/include/asm/pgtable-ppc64.h | 31 +---- arch/powerpc/kernel/io-workarounds.c | 3 +- arch/powerpc/kvm/book3s_64_mmu_hv.c | 2 +- arch/powerpc/kvm/book3s_hv_rm_mmu.c | 4 +- arch/powerpc/mm/Makefile | 1 + arch/powerpc/mm/hash_utils_64.c | 16 ++- arch/powerpc/mm/hugepage-hash64.c | 185 ++++++++++++++++++++++++++++++ arch/powerpc/mm/hugetlbpage.c | 31 ++++- arch/powerpc/mm/pgtable.c | 38 ++++++ arch/powerpc/mm/tlb_hash64.c | 5 +- arch/powerpc/perf/callchain.c | 2 +- arch/powerpc/platforms/pseries/eeh.c | 5 +- 13 files changed, 286 insertions(+), 42 deletions(-) create mode 100644 arch/powerpc/mm/hugepage-hash64.c
arch/powerpc/include/asm/mmu-hash64.h +5 -0
@@ -322,6 +322,11 @@ extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap)int __hash_page_huge (unsigned long ea,unsigned long access,unsigned long vsid,pte_t *ptep,unsigned long trap,int local,int ssize,unsigned int shift,unsigned int mmu_psize); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int __hash_page_thp (unsigned long ea,unsigned long access, +unsigned long vsid,pmd_t *pmdp,unsigned long trap, +int local,int ssize,unsigned int psize); +#endif extern void hash_failure_debug(unsigned long ea, unsigned long access, unsigned long vsid, unsigned long trap, int ssize, int psize, int lpsize,
arch/powerpc/include/asm/pgtable-ppc64.h +5 -26
@@ -345,39 +345,18 @@ static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)void pgtable_cache_add (unsigned shift,void (*ctor)(void *));void pgtable_cache_init (void ); -/* -* find_linux_pte returns the address of a linux pte for a given -* effective address and directory. If not found, it returns zero. -*/ -static inline pte_t *find_linux_pte (pgd_t *pgdir,unsigned long ea) -{ -pgd_t *pg; -pud_t *pu; -pmd_t *pm; -pte_t *pt =NULL ; - - pg = pgdir +pgd_index (ea); -if (!pgd_none (*pg)) { - pu =pud_offset (pg, ea); -if (!pud_none (*pu)) { - pm =pmd_offset (pu, ea); -if (pmd_present (*pm)) - pt =pte_offset_kernel (pm, ea); - } - } -return pt; -} - +pte_t *find_linux_pte (pgd_t *pgdir,unsigned long ea,unsigned int *thp);#ifdef CONFIG_HUGETLB_PAGE pte_t *find_linux_pte_or_hugepte (pgd_t *pgdir,unsigned long ea, -unsigned *shift); +unsigned *shift,unsigned int *hugepage);#else static inline pte_t *find_linux_pte_or_hugepte (pgd_t *pgdir,unsigned long ea, -unsigned *shift) +unsigned *shift, +unsigned int *hugepage) {if (shift) *shift =0 ; -return find_linux_pte (pgdir, ea); +return find_linux_pte (pgdir, ea, hugepage); }#endif /* !CONFIG_HUGETLB_PAGE */
arch/powerpc/kernel/io-workarounds.c +2 -1
@@ -70,7 +70,8 @@ struct iowa_bus *iowa_mem_find_bus(const PCI_IO_ADDR addr)if (vaddr <PHB_IO_BASE || vaddr >=PHB_IO_END )return NULL ; - ptep =find_linux_pte (init_mm.pgd, vaddr); +/* we won't find hugepages here */ + ptep =find_linux_pte (init_mm.pgd, vaddr,NULL );if (ptep ==NULL ) paddr =0 ;else
arch/powerpc/kvm/book3s_64_mmu_hv.c +1 -1
@@ -683,7 +683,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,*/ rcu_read_lock_sched (); ptep =find_linux_pte_or_hugepte (current->mm->pgd, - hva,NULL ); + hva,NULL ,NULL );if (ptep &&pte_present (*ptep)) { pte =kvmppc_read_update_linux_pte (ptep,1 );if (pte_write (pte))
arch/powerpc/kvm/book3s_hv_rm_mmu.c +2 -2
@@ -27,7 +27,7 @@ static void *real_vmalloc_addr(void *x)unsigned long addr = (unsigned long ) x;pte_t *p; - p =find_linux_pte (swapper_pg_dir, addr); + p =find_linux_pte (swapper_pg_dir, addr,NULL );if (!p || !pte_present (*p))return NULL ;/* assume we don't have huge pages in vmalloc space... */
@@ -152,7 +152,7 @@ static pte_t lookup_linux_pte(pgd_t *pgdir, unsigned long hva,unsigned long ps = *pte_sizep;unsigned int shift; - ptep =find_linux_pte_or_hugepte (pgdir, hva, &shift); + ptep =find_linux_pte_or_hugepte (pgdir, hva, &shift,NULL );if (!ptep)return __pte (0 );if (shift)
arch/powerpc/mm/Makefile +1 -0
@@ -33,6 +33,7 @@ obj-y += hugetlbpage.o obj-$(CONFIG_PPC_STD_MMU_64) += hugetlbpage-hash64.o obj-$(CONFIG_PPC_BOOK3E_MMU) += hugetlbpage-book3e.o endif +obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += hugepage-hash64.o obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage-prot.o obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o obj-$(CONFIG_HIGHMEM) += highmem.o
arch/powerpc/mm/hash_utils_64.c +13 -3
@@ -955,7 +955,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)unsigned long vsid;struct mm_struct *mm;pte_t *ptep; -unsigned hugeshift; +unsigned hugeshift, hugepage;const struct cpumask *tmp;int rc, user_region =0 , local =0 ;int psize, ssize;
@@ -1021,7 +1021,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)#endif /* CONFIG_PPC_64K_PAGES */ /* Get PTE and page size from page tables */ - ptep =find_linux_pte_or_hugepte (pgdir, ea, &hugeshift); + ptep =find_linux_pte_or_hugepte (pgdir, ea, &hugeshift, &hugepage);if (ptep ==NULL || !pte_present (*ptep)) {DBG_LOW (" no PTE !\n" );return 1 ;
@@ -1044,6 +1044,12 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) ssize, hugeshift, psize);#endif /* CONFIG_HUGETLB_PAGE */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +if (hugepage) +return __hash_page_thp (ea, access, vsid, (pmd_t *)ptep, + trap, local, ssize, psize); +#endif +#ifndef CONFIG_PPC_64K_PAGES DBG_LOW (" i-pte: %016lx\n" ,pte_val (*ptep));#else
@@ -1149,7 +1155,11 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, pgdir = mm->pgd;if (pgdir ==NULL )return ; - ptep =find_linux_pte (pgdir, ea); +/* +* We haven't implemented update_mmu_cache_pmd yet. We get called +* only for non hugepages. Hence can ignore THP here +*/ + ptep =find_linux_pte (pgdir, ea,NULL );if (!ptep)return ;
arch/powerpc/mm/hugepage-hash64.c new file +185 -0
185 lines not on this page · raw message
arch/powerpc/mm/hugetlbpage.c +25 -6
@@ -67,7 +67,8 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)#define hugepd_none (hpd) ((hpd).pd == 0) -pte_t *find_linux_pte_or_hugepte (pgd_t *pgdir,unsigned long ea,unsigned *shift) +pte_t *find_linux_pte_or_hugepte (pgd_t *pgdir,unsigned long ea, +unsigned *shift,unsigned int *hugepage) {pgd_t *pg;pud_t *pu;
@@ -77,6 +78,8 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shiftif (shift) *shift =0 ; +if (hugepage) + *hugepage =0 ; pg = pgdir +pgd_index (ea);if (is_hugepd (pg)) {
@@ -91,12 +94,24 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift pm =pmd_offset (pu, ea);if (is_hugepd (pm)) hpdp = (hugepd_t *)pm; -else if (!pmd_none (*pm)) { +else if (pmd_large (*pm)) { +/* THP page */ +if (hugepage) { + *hugepage =1 ; +/* +* This should be ok, except for few +* flags. Most of the pte and hugepage +* pmd bits overlap. We don't use the +* returned value as pte_t in the caller. +*/ +return (pte_t *)pm; + }else +return NULL ; + }else if (!pmd_none (*pm)) {return pte_offset_kernel (pm, ea); } } } -if (!hpdp)return NULL ;
@@ -108,7 +123,8 @@ EXPORT_SYMBOL_GPL(find_linux_pte_or_hugepte);pte_t *huge_pte_offset (struct mm_struct *mm,unsigned long addr) { -return find_linux_pte_or_hugepte (mm->pgd, addr,NULL ); +/* Only called for HugeTLB pages, hence can ignore THP */ +return find_linux_pte_or_hugepte (mm->pgd, addr,NULL ,NULL ); } static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
@@ -613,8 +629,11 @@ follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)struct page *page;unsigned shift;unsigned long mask; - - ptep =find_linux_pte_or_hugepte (mm->pgd, address, &shift); +/* +* Transparent hugepages are handled by generic code. We can skip them +* here. +*/ + ptep =find_linux_pte_or_hugepte (mm->pgd, address, &shift,NULL );/* Verify it is a huge page else bail. */ if (!ptep || !shift)
arch/powerpc/mm/pgtable.c +38 -0
@@ -557,3 +557,41 @@ void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, }#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +/* +* find_linux_pte returns the address of a linux pte for a given +* effective address and directory. If not found, it returns zero. +*/ +pte_t *find_linux_pte (pgd_t *pgdir,unsigned long ea,unsigned int *hugepage) +{ +pgd_t *pg; +pud_t *pu; +pmd_t *pm; +pte_t *pt =NULL ; + +if (hugepage) + *hugepage =0 ; + pg = pgdir +pgd_index (ea); +if (!pgd_none (*pg)) { + pu =pud_offset (pg, ea); +if (!pud_none (*pu)) { + pm =pmd_offset (pu, ea); +if (pmd_large (*pm)) { +/* THP page */ +if (hugepage) { + *hugepage =1 ; +/* +* This should be ok, except for few +* flags. Most of the pte and hugepage +* pmd bits overlap. We don't use the +* returned value as pte_t in the caller. +*/ +return (pte_t *)pm; + }else +return NULL ; + }else if (pmd_present (*pm)) + pt =pte_offset_kernel (pm, ea); + } + } +return pt; +}
arch/powerpc/mm/tlb_hash64.c +4 -1
@@ -206,7 +206,10 @@ void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,local_irq_save (flags);arch_enter_lazy_mmu_mode ();for (; start < end; start +=PAGE_SIZE ) { -pte_t *ptep =find_linux_pte (mm->pgd, start); +/* +* We won't find hugepages here. +*/ +pte_t *ptep =find_linux_pte (mm->pgd, start,NULL );unsigned long pte;if (ptep ==NULL )
arch/powerpc/perf/callchain.c +1 -1
@@ -125,7 +125,7 @@ static int read_user_stack_slow(void __user *ptr, void *ret, int nb)if (!pgdir)return -EFAULT ; - ptep =find_linux_pte_or_hugepte (pgdir, addr, &shift); + ptep =find_linux_pte_or_hugepte (pgdir, addr, &shift,NULL );if (!shift) shift =PAGE_SHIFT ;
arch/powerpc/platforms/pseries/eeh.c +4 -1
@@ -261,7 +261,10 @@ static inline unsigned long eeh_token_to_phys(unsigned long token)pte_t *ptep;unsigned long pa; - ptep =find_linux_pte (init_mm.pgd, token); +/* +* We won't find hugepages here +*/ + ptep =find_linux_pte (init_mm.pgd, token,NULL );if (!ptep)return token; pa =pte_pfn (*ptep) <<PAGE_SHIFT ;
Signature · 1 line
1.7.10
PATCH 21/25 powerpc: Handle hugepage in perf callchain
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
--- arch/powerpc/perf/callchain.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-)
arch/powerpc/perf/callchain.c +21 -11
@@ -115,7 +115,7 @@ static int read_user_stack_slow(void __user *ptr, void *ret, int nb) {pgd_t *pgdir;pte_t *ptep, pte; -unsigned shift; +unsigned shift, hugepage;unsigned long addr = (unsigned long ) ptr;unsigned long offset;unsigned long pfn;
@@ -125,20 +125,30 @@ static int read_user_stack_slow(void __user *ptr, void *ret, int nb)if (!pgdir)return -EFAULT ; - ptep =find_linux_pte_or_hugepte (pgdir, addr, &shift,NULL ); + ptep =find_linux_pte_or_hugepte (pgdir, addr, &shift, &hugepage);if (!shift) shift =PAGE_SHIFT ; -/* align address to page boundary */ - offset = addr & ((1UL << shift) -1 ); - addr -= offset; - -if (ptep ==NULL ) -return -EFAULT ; - pte = *ptep; -if (!pte_present (pte) || !(pte_val (pte) & _PAGE_USER)) +if (!ptep)return -EFAULT ; - pfn =pte_pfn (pte); + +if (hugepage) { +pmd_t pmd = *(pmd_t *)ptep; + shift = mmu_psize_defs[MMU_PAGE_16M ].shift; + offset = addr & ((1UL << shift) -1 ); + +if (!pmd_large (pmd) || !(pmd_val (pmd) &PMD_HUGE_USER )) +return -EFAULT ; + pfn =pmd_pfn (pmd); + }else { + offset = addr & ((1UL << shift) -1 ); + + pte = *ptep; +if (!pte_present (pte) || !(pte_val (pte) & _PAGE_USER)) +return -EFAULT ; + pfn =pte_pfn (pte); + } +if (!page_is_ram (pfn))return -EFAULT ;
Signature · 1 line
1.7.10
PATCH 22/25 powerpc/THP: get_user_pages_fast changes
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
handle large pages for get_user_pages_fast. Also take care of large page splitting.
--- arch/powerpc/mm/gup.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-)
arch/powerpc/mm/gup.c +82 -2
@@ -55,6 +55,72 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,return 1 ; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int gup_huge_pmd (pmd_t *pmdp,unsigned long addr, +unsigned long end,int write, +struct page **pages,int *nr) +{ +int refs; +pmd_t pmd; +unsigned long mask; +struct page *head, *page, *tail; + + pmd = *pmdp; + mask =PMD_HUGE_PRESENT |PMD_HUGE_USER ; +if (write) + mask |=PMD_HUGE_RW ; + +if ((pmd_val (pmd) & mask) != mask) +return 0 ; + +/* large pages are never "special" */ +VM_BUG_ON (!pfn_valid (pmd_pfn (pmd))); + + refs =0 ; + head =pmd_page (pmd); + page = head + ((addr & ~PMD_MASK ) >>PAGE_SHIFT ); + tail = page; +do { +VM_BUG_ON (compound_head (page) != head); + pages[*nr] = page; + (*nr)++; + page++; + refs++; + }while (addr +=PAGE_SIZE , addr != end); + +if (!page_cache_add_speculative (head, refs)) { + *nr -= refs; +return 0 ; + } + +if (unlikely (pmd_val (pmd) !=pmd_val (*pmdp))) { + *nr -= refs; +while (refs--) +put_page (head); +return 0 ; + } +/* +* Any tail page need their mapcount reference taken before we +* return. +*/ +while (refs--) { +if (PageTail (tail)) +get_huge_page_tail (tail); + tail++; + } + +return 1 ; +} +#else + +static inline int gup_huge_pmd (pmd_t *pmdp,unsigned long addr, +unsigned long end,int write, +struct page **pages,int *nr) +{ +return 1 ; +} +#endif +static int gup_pmd_range (pud_t pud,unsigned long addr,unsigned long end,int write,struct page **pages,int *nr) {
@@ -66,9 +132,23 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,pmd_t pmd = *pmdp; next =pmd_addr_end (addr, end); -if (pmd_none (pmd)) +/* +* The pmd_trans_splitting() check below explains why +* pmdp_splitting_flush has to flush the tlb, to stop +* this gup-fast code from running while we set the +* splitting bit in the pmd. Returning zero will take +* the slow path that will call wait_split_huge_page() +* if the pmd is still in splitting state. gup-fast +* can't because it has irq disabled and +* wait_split_huge_page() would never return as the +* tlb flush IPI wouldn't run. +*/ +if (pmd_none (pmd) ||pmd_trans_splitting (pmd))return 0 ; -if (is_hugepd (pmdp)) { +if (unlikely (pmd_large (pmd))) { +if (!gup_huge_pmd (pmdp, addr, next, write, pages, nr)) +return 0 ; + }else if (is_hugepd (pmdp)) {if (!gup_hugepd ((hugepd_t *)pmdp,PMD_SHIFT , addr, next, write, pages, nr))return 0 ;
Signature · 1 line
1.7.10
PATCH 23/25 powerpc/THP: Enable THP on PPC64
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We enable only if the we support 16MB page size.
--- arch/powerpc/include/asm/pgtable.h | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-)
arch/powerpc/include/asm/pgtable.h +29 -2
@@ -81,8 +81,35 @@ static inline int pmd_trans_huge(pmd_t pmd) return ((pmd_val(pmd) & PMD_ISHUGE) == PMD_ISHUGE); } -/* We will enable it in the last patch */ -#define has_transparent_hugepage () 0 +static inline int has_transparent_hugepage (void ) +{ +if (!mmu_has_feature (MMU_FTR_16M_PAGE )) +return 0 ; +/* +* We support THP only if HPAGE_SHIFT is 16MB. +*/ +if (!HPAGE_SHIFT || (HPAGE_SHIFT != mmu_psize_defs[MMU_PAGE_16M ].shift)) +return 0 ; +/* +* We need to make sure that we support 16MB hugepage in a segement +* with base page size 64K or 4K. We only enable THP with a PAGE_SIZE +* of 64K. +*/ +/* +* If we have 64K HPTE, we will be using that by default +*/ +if (mmu_psize_defs[MMU_PAGE_64K ].shift && + (mmu_psize_defs[MMU_PAGE_64K ].penc[MMU_PAGE_16M ] ==-1 )) +return 0 ; +/* +* Ok we only have 4K HPTE +*/ +if (mmu_psize_defs[MMU_PAGE_4K ].penc[MMU_PAGE_16M ] ==-1 ) +return 0 ; + +return 1 ; +} +#else #define pmd_large (pmd) 0#define has_transparent_hugepage () 0
Signature · 1 line
1.7.10
PATCH 24/25 powerpc: Optimize hugepage invalidate
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
Hugepage invalidate involves invalidating multiple hpte entries. Optimize the operation using H_BULK_REMOVE on lpar platforms. On native, reduce the number of tlb flush.
--- arch/powerpc/include/asm/machdep.h | 3 + arch/powerpc/mm/hash_native_64.c | 78 ++++++++++++++++++++ arch/powerpc/mm/pgtable.c | 13 +++- arch/powerpc/platforms/pseries/lpar.c | 126 +++++++++++++++++++++++++++++++-- 4 files changed, 210 insertions(+), 10 deletions(-)
arch/powerpc/include/asm/machdep.h +3 -0
@@ -56,6 +56,9 @@ struct machdep_calls {void (*hpte_removebolted)(unsigned long ea,int psize,int ssize);void (*flush_hash_range)(unsigned long number,int local); +void (*hugepage_invalidate)(struct mm_struct *mm, +unsigned char *hpte_slot_array, +unsigned long addr,int psize);/* special for kexec, to be called in real mode, linear mapping is * destroyed as well */
arch/powerpc/mm/hash_native_64.c +78 -0
88 lines not on this page · raw message
arch/powerpc/mm/pgtable.c +10 -3
@@ -433,6 +433,7 @@ void hpte_need_hugepage_flush(struct mm_struct *mm, unsigned long addr, {int ssize, i;unsigned long s_addr; +int max_hpte_count;unsigned int psize, valid;unsigned char *hpte_slot_array;unsigned long hidx, vpn, vsid, hash, shift, slot;
@@ -446,12 +447,18 @@ void hpte_need_hugepage_flush(struct mm_struct *mm, unsigned long addr,* second half of the PMD */ hpte_slot_array = *(char **)(pmdp +PTRS_PER_PMD ); -/* get the base page size */ psize =get_slice_psize (mm, s_addr); - shift = mmu_psize_defs[psize].shift; -for (i =0 ; i <HUGE_PAGE_SIZE /(1ul << shift); i++) { +if (ppc_md.hugepage_invalidate) +return ppc_md.hugepage_invalidate (mm, hpte_slot_array, + s_addr, psize); +/* +* No bluk hpte removal support, invalidate each entry +*/ + shift = mmu_psize_defs[psize].shift; + max_hpte_count =HUGE_PAGE_SIZE /(1ul << shift); +for (i =0 ; i < max_hpte_count; i++) {/* * 8 bits per each hpte entries * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]
arch/powerpc/platforms/pseries/lpar.c +119 -7
150 lines not on this page · raw message
Signature · 1 line
1.7.10
PATCH 25/25 powerpc: Handle hugepages in kvm
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We could possibly avoid some of these changes because most of the HUGE PMD bits map to PTE bits.
--- arch/powerpc/include/asm/kvm_book3s_64.h | 31 ++++++++++++ arch/powerpc/kvm/book3s_64_mmu_hv.c | 12 ++++- arch/powerpc/kvm/book3s_hv_rm_mmu.c | 75 ++++++++++++++++++++++-------- 3 files changed, 97 insertions(+), 21 deletions(-)
arch/powerpc/include/asm/kvm_book3s_64.h +31 -0
@@ -110,6 +110,7 @@ static inline unsigned long compute_tlbie_rb(unsigned long v, unsigned long r,return rb; } +/* FIXME !! should we use hpte_actual_psize or hpte decode ? */ static inline unsigned long hpte_page_size (unsigned long h,unsigned long l) {/* only handle 4k, 64k and 16M pages for now */
@@ -189,6 +190,36 @@ static inline pte_t kvmppc_read_update_linux_pte(pte_t *p, int writing)return pte; } +/* +* Lock and read a linux hugepage PMD. If it's present and writable, atomically +* set dirty and referenced bits and return the PMD, otherwise return 0. +*/ +static inline pmd_t kvmppc_read_update_linux_hugepmd (pmd_t *p,int writing) +{ +pmd_t pmd, tmp; + +/* wait until _PAGE_BUSY is clear then set it atomically */ + __asm__ __volatile__ ( +"1: ldarx %0,0,%3\n" +" andi. %1,%0,%4\n" +" bne- 1b\n" +" ori %1,%0,%4\n" +" stdcx. %1,0,%3\n" +" bne- 1b" + :"=&r" (pmd),"=&r" (tmp),"=m" (*p) + :"r" (p),"i" (PMD_HUGE_BUSY ) + :"cc" ); + +if (pmd_large (pmd)) { + pmd =pmd_mkyoung (pmd); +if (writing &&pmd_write (pmd)) + pmd =pte_mkdirty (pmd); + } + + *p = pmd;/* clears PMD_HUGE_BUSY */ +return pmd; +} +/* Return HPTE cache control bits corresponding to Linux pte bits */ static inline unsigned long hpte_cache_bits (unsigned long pte_val) {
arch/powerpc/kvm/book3s_64_mmu_hv.c +10 -2
@@ -675,6 +675,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, }/* if the guest wants write access, see if that is OK */ if (!writing &&hpte_is_writable (r)) { +int hugepage;pte_t *ptep, pte;/*
@@ -683,11 +684,18 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,*/ rcu_read_lock_sched (); ptep =find_linux_pte_or_hugepte (current->mm->pgd, - hva,NULL ,NULL ); -if (ptep &&pte_present (*ptep)) { + hva,NULL , &hugepage); +if (!hugepage && ptep &&pte_present (*ptep)) { pte =kvmppc_read_update_linux_pte (ptep,1 );if (pte_write (pte)) write_ok =1 ; + }else if (hugepage && ptep) { +pmd_t pmd = *(pmd_t *)ptep; +if (pmd_large (pmd)) { + pmd =kvmppc_read_update_linux_hugepmd ((pmd_t *)ptep,1 ); +if (pmd_write (pmd)) + write_ok =1 ; + } }rcu_read_unlock_sched (); }
arch/powerpc/kvm/book3s_hv_rm_mmu.c +56 -19
108 lines not on this page · raw message
Signature · 1 line
1.7.10