powerpc: mm: Numa faults support for ppc64
Subsystems (5)
cover powerpc: mm: Numa faults support for ppc64
Hi,
This patch series add support for numa faults on ppc64 architecture. We steal the _PAGE_COHERENCE bit and use that for indicating _PAGE_NUMA. We clear the _PAGE_PRESENT bit and also invalidate the hpte entry on setting _PAGE_NUMA. The next fault on that page will be considered a numa fault.
Changes from V1:
* Dropped few patches related pmd update because batch handling of pmd pages got dropped from core code 0f19c17929c952c6f0966d93ab05558e7bf814cc "mm: numa: Do not batch handle PMD pages"
This also avoided the large lock contention on page_table_lock that we observed with the previous series.
-aneesh
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 1/5 powerpc: Use HPTE constants when updating hpte bits
From: "Aneesh Kumar K.V" <[email protected]>
Even though we have same value for linux PTE bits and hash PTE pits use the hash pte bits wen updating hash pte
--- arch/powerpc/platforms/cell/beat_htab.c | 4 ++-- arch/powerpc/platforms/pseries/lpar.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-)
arch/powerpc/platforms/cell/beat_htab.c +2 -2
@@ -111,7 +111,7 @@ static long beat_lpar_hpte_insert(unsigned long hpte_group,DBG_LOW (" hpte_v=%016lx, hpte_r=%016lx\n" , hpte_v, hpte_r);if (rflags & _PAGE_NO_CACHE) - hpte_r &= ~_PAGE_COHERENT; + hpte_r &= ~HPTE_R_M ;raw_spin_lock (&beat_htab_lock); lpar_rc =beat_read_mask (hpte_group);
@@ -337,7 +337,7 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,DBG_LOW (" hpte_v=%016lx, hpte_r=%016lx\n" , hpte_v, hpte_r);if (rflags & _PAGE_NO_CACHE) - hpte_r &= ~_PAGE_COHERENT; + hpte_r &= ~HPTE_R_M ;/* insert into not-volted entry */ lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
arch/powerpc/platforms/pseries/lpar.c +2 -1
@@ -153,7 +153,8 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,/* Make pHyp happy */ if ((rflags & _PAGE_NO_CACHE) && !(rflags & _PAGE_WRITETHRU)) - hpte_r &= ~_PAGE_COHERENT; + hpte_r &= ~HPTE_R_M ; +if (firmware_has_feature (FW_FEATURE_XCMO ) && !(hpte_r &HPTE_R_N )) flags |=H_COALESCE_CAND ;
Signature · 1 line
1.8.3.2
Re: [PATCH -V2 1/5] powerpc: Use HPTE constants when updating hpte bits
Quoted 3 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
Even though we have same value for linux PTE bits and hash PTE pits
bits, not pits :)
Quoted 1 line
use the hash pte bits wen updating hash pte
when, not wen
Quoted 1 line
Signed-off-by Aneesh Kumar K.V <[email protected]>
If you fix the spelling errors in the patch description:
PATCH 2/5 powerpc: Free up _PAGE_COHERENCE for numa fault use later
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
Set memory coherence always on hash64 config. If a platform cannot have memory coherence always set they can infer that from _PAGE_NO_CACHE and _PAGE_WRITETHRU like in lpar. So we dont' really need a separate bit for tracking _PAGE_COHERENCE.
--- arch/powerpc/include/asm/pte-hash64.h | 2 +- arch/powerpc/mm/hash_low_64.S | 15 ++++++++++++--- arch/powerpc/mm/hash_utils_64.c | 7 ++++--- arch/powerpc/mm/hugepage-hash64.c | 6 +++++- arch/powerpc/mm/hugetlbpage-hash64.c | 4 ++++ 5 files changed, 26 insertions(+), 8 deletions(-)
arch/powerpc/include/asm/pte-hash64.h +1 -1
@@ -19,7 +19,7 @@#define _PAGE_FILE 0x0002/* (!present only) software: pte holds file offset */ #define _PAGE_EXEC 0x0004/* No execute on POWER4 and newer (we invert) */ #define _PAGE_GUARDED 0x0008 -#define _PAGE_COHERENT 0x0010/* M: enforce memory coherence (SMP systems) */ +/* We can derive Memory coherence from _PAGE_NO_CACHE */ #define _PAGE_NO_CACHE 0x0020/* I: cache inhibit */ #define _PAGE_WRITETHRU 0x0040/* W: cache write-through */ #define _PAGE_DIRTY 0x0080/* C: page changed */
arch/powerpc/mm/hash_low_64.S +12 -3
@@ -148,7 +148,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r30,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...)
@@ -457,7 +460,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r3,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...)
@@ -795,7 +801,10 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ andc r0,r30,r0 /* r0 = pte & ~r0 */ rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ - ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ + /* + * Always add "C" bit for perf. Memory coherence is always enabled + */ + ori r3,r3,HPTE_R_C | HPTE_R_M /* We eventually do the icache sync here (maybe inline that * code rather than call a C function...)
arch/powerpc/mm/hash_utils_64.c +4 -3
@@ -169,9 +169,10 @@ static unsigned long htab_convert_pte_flags(unsigned long pteflags)if ((pteflags & _PAGE_USER) && !((pteflags & _PAGE_RW) && (pteflags & _PAGE_DIRTY))) rflags |=1 ; - -/* Always add C */ -return rflags |HPTE_R_C ; +/* +* Always add "C" bit for perf. Memory coherence is always enabled +*/ +return rflags |HPTE_R_C |HPTE_R_M ; } int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
arch/powerpc/mm/hugepage-hash64.c +5 -1
@@ -127,7 +127,11 @@ repeat:/* Add in WIMG bits */ rflags |= (new_pmd & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | - _PAGE_COHERENT | _PAGE_GUARDED)); + _PAGE_GUARDED)); +/* +* enable the memory coherence always +*/ + rflags |=HPTE_R_M ;/* Insert into the hash table, primary slot */ slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
arch/powerpc/mm/hugetlbpage-hash64.c +4 -0
@@ -99,6 +99,10 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,/* Add in WIMG bits */ rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | _PAGE_COHERENT | _PAGE_GUARDED)); +/* +* enable the memory coherence always +*/ + rflags |=HPTE_R_M ; slot =hpte_insert_repeating (hash, vpn, pa, rflags,0 , mmu_psize, ssize);
Signature · 1 line
1.8.3.2
Re: [PATCH -V2 2/5] powerpc: Free up _PAGE_COHERENCE for numa fault use later
Quoted 9 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
Set memory coherence always on hash64 config. If a platform cannot have memory coherence always set they can infer that from
_PAGE_NO_CACHEand_PAGE_WRITETHRUlike in lpar. So we dont' really need a separate bit for tracking_PAGE_COHERENCE.Signed-off-by Aneesh Kumar K.V <[email protected]>
PATCH 3/5 mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
change_prot_numa should work even if _PAGE_NUMA != _PAGE_PROTNONE. On archs like ppc64 that don't use _PAGE_PROTNONE and also have a separate page table outside linux pagetable, we just need to make sure that when calling change_prot_numa we flush the hardware page table entry so that next page access result in a numa fault.
--- include/linux/mm.h | 3 --- mm/mempolicy.c | 9 --------- 2 files changed, 12 deletions(-)
include/linux/mm.h +0 -3
@@ -1851,11 +1851,8 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif -#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end); -#endif -struct vm_area_struct *find_extend_vma (struct mm_struct *,unsigned long addr);int remap_pfn_range (struct vm_area_struct *,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t );
mm/mempolicy.c +0 -9
@@ -613,7 +613,6 @@ static inline int queue_pages_pgd_range(struct vm_area_struct *vma,return 0 ; } -#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE /* * This is used to mark a range of virtual addresses to be inaccessible. * These are later cleared by a NUMA hinting fault. Depending on these
@@ -627,7 +626,6 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, unsigned long addr, unsigned long end) { int nr_updated; -BUILD_BUG_ON (_PAGE_NUMA != _PAGE_PROTNONE); nr_updated =change_protection (vma, addr, end, vma->vm_page_prot,0 ,1 );if (nr_updated)
@@ -635,13 +633,6 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,return nr_updated; } -#else -static unsigned long change_prot_numa (struct vm_area_struct *vma, -unsigned long addr,unsigned long end) -{ -return 0 ; -} -#endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */ /* * Walk through page tables and collect pages to be migrated.
Signature · 1 line
1.8.3.2
Re: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
Quoted 8 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
change_prot_numashould work even if_PAGE_NUMA!=_PAGE_PROTNONE. On archs like ppc64 that don't use_PAGE_PROTNONEand also have a separate page table outside linux pagetable, we just need to make sure that when callingchange_prot_numawe flush the hardware page table entry so that next page access result in a numa fault.
That patch doesn't look right...
You are essentially making change_prot_numa() do whatever it does (which I don't completely understand) *for all architectures* now, whether they have CONFIG_ARCH_USES_NUMA_PROT_NONE or not ... So because you want that behaviour on powerpc book3s64, you change everybody.
Is that correct ?
Also what exactly is that doing, can you explain ? From what I can see, it calls back into the core of mprotect to change the protection to vma->vm_page_prot, which I would have expected is already the protection there, with the added "prot_numa" flag passed down.
Your changeset comment says "On archs like ppc64 [...] we just need to make sure that when calling change_prot_numa we flush the hardware page table entry so that next page access result in a numa fault."
But change_prot_numa() does a lot more than that ... it does pte_mknuma(), do we need it ? I assume we do or we wouldn't have added that PTE bit to begin with...
Now it *might* be allright and it might be that no other architecture cares anyway etc... but I need at least some mm folks to ack on that patch before I can take it because it *will* change behaviour of other architectures.
Cheers,
Ben.
Quoted 56 lines
Signed-off-by Aneesh Kumar K.V <[email protected]>--- include/linux/mm.h | 3 --- mm/mempolicy.c | 9 --------- 2 files changed, 12 deletions(-)include/linux/mm.h +0
-3@@ -1851,11 +1851,8 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif -#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end); -#endif -struct vm_area_struct *find_extend_vma (struct mm_struct *,unsigned long addr);int remap_pfn_range (struct vm_area_struct *,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t );mm/mempolicy.c +0
-9@@ -613,7 +613,6 @@ static inline int queue_pages_pgd_range(struct vm_area_struct *vma,return 0 ; } -#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE /* * This is used to mark a range of virtual addresses to be inaccessible. * These are later cleared by a NUMA hinting fault. Depending on these @@ -627,7 +626,6 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, unsigned long addr, unsigned long end) { int nr_updated; -BUILD_BUG_ON (_PAGE_NUMA != _PAGE_PROTNONE); nr_updated =change_protection (vma, addr, end, vma->vm_page_prot,0 ,1 );if (nr_updated)@@ -635,13 +633,6 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,return nr_updated; } -#else -static unsigned long change_prot_numa (struct vm_area_struct *vma, -unsigned long addr,unsigned long end) -{ -return 0 ; -} -#endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */ /* * Walk through page tables and collect pages to be migrated.
Re: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
Adding Mel and Rik to cc:
Quoted 10 lines · Benjamin Herrenschmidt
Quoted 8 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
change_prot_numashould work even if_PAGE_NUMA!=_PAGE_PROTNONE. On archs like ppc64 that don't use_PAGE_PROTNONEand also have a separate page table outside linux pagetable, we just need to make sure that when callingchange_prot_numawe flush the hardware page table entry so that next page access result in a numa fault.
Quoted 8 lines
That patch doesn't look right...
You are essentially making
change_prot_numa()do whatever it does (which I don't completely understand) *for all architectures* now, whether they haveCONFIG_ARCH_USES_NUMA_PROT_NONEor not ... So because you want that behaviour on powerpc book3s64, you change everybody.Is that correct ?
Yes.
Quoted 5 lines
Also what exactly is that doing, can you explain ? From what I can see, it calls back into the core of mprotect to change the protection to
vma->vm_page_prot, which I would have expected is already the protection there, with the added "prot_numa" flag passed down.
it set the _PAGE_NUMA bit. Now we also want to make sure that when we set _PAGE_NUMA, we would get a pagefault on that so that we can track that fault as a numa fault. To ensure that, we had the below BUILD_BUG
BUILD_BUG_ON(_PAGE_NUMA != _PAGE_PROTNONE);
But other than that the function doesn't really have any dependency on _PAGE_PROTNONE. The only requirement is when we set _PAGE_NUMA, the architecture should do enough to ensure that we get a page fault. Now on ppc64 we does that by clearlying hpte entry and also clearing _PAGE_PRESENT. Since we have _PAGE_PRESENT cleared hash_page will return 1 and we get to page fault handler.
Quoted 6 lines
Your changeset comment says "On archs like ppc64 [...] we just need to make sure that when calling
change_prot_numawe flush the hardware page table entry so that next page access result in a numa fault."
Quoted 9 lines
But
change_prot_numa()does a lot more than that ... it doespte_mknuma(), do we need it ? I assume we do or we wouldn't have added that PTE bit to begin with...Now it *might* be allright and it might be that no other architecture cares anyway etc... but I need at least some mm folks to ack on that patch before I can take it because it *will* change behaviour of other architectures.
Ok, I can move the changes below #ifdef CONFIG_NUMA_BALANCING ? We call change_prot_numa from task_numa_work and queue_pages_range(). The later may be an issue. So doing the below will help ?
-#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
+#ifdef CONFIG_NUMA_BALANCING
-aneesh
Re: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
Review
Quoted 7 lines · Aneesh Kumar K.V
Ok, I can move the changes below #ifdef
CONFIG_NUMA_BALANCING? We callchange_prot_numafromtask_numa_workandqueue_pages_range(). The later may be an issue. So doing the below will help ?(quoted diff) +1
-1-#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE +#ifdef CONFIG_NUMA_BALANCING
I will defer to Mel and Rik (should we also CC Andrea ?)
Cheers,
Ben.
Re: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
Quoted 10 lines · Benjamin Herrenschmidt
Review
Quoted 7 lines · Aneesh Kumar K.V
Ok, I can move the changes below #ifdef
CONFIG_NUMA_BALANCING? We callchange_prot_numafromtask_numa_workandqueue_pages_range(). The later may be an issue. So doing the below will help ?(quoted diff) +1
-1-#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE +#ifdef CONFIG_NUMA_BALANCINGI will defer to Mel and Rik (should we also CC Andrea ?)
It looks like manual numa binding can also use lazy page migration, but I am not sure if that can happen without CONFIG_NUMA_BALANCING, or if mbind always uses MPOL_MF_STRICT...
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Rik van Riel
Reviewed-byRe: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
In reply to Benjamin HerrenschmidtQuoted 11 lines · Benjamin Herrenschmidt
Quoted 8 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
change_prot_numashould work even if_PAGE_NUMA!=_PAGE_PROTNONE. On archs like ppc64 that don't use_PAGE_PROTNONEand also have a separate page table outside linux pagetable, we just need to make sure that when callingchange_prot_numawe flush the hardware page table entry so that next page access result in a numa fault.That patch doesn't look right...
At first glance, indeed...
Quoted 4 lines
You are essentially making
change_prot_numa()do whatever it does (which I don't completely understand) *for all architectures* now, whether they haveCONFIG_ARCH_USES_NUMA_PROT_NONEor not ... So because you want that behaviour on powerpc book3s64, you change everybody.
However, it appears that since the code was #ifdefed like that, the called code was made generic enough, that change_prot_numa should actually work for everything.
In other words:
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH -V2 3/5] mm: Move change_prot_numa outside CONFIG_ARCH_USES_NUMA_PROT_NONE
Quoted 8 lines · Rik van Riel
However, it appears that since the code was #ifdefed like that, the called code was made generic enough, that
change_prot_numashould actually work for everything.In other words:
Reviewed-by Rik van Riel <[email protected]>
Ok thanks, that's what I needed. Do you have any objection of me merging that change via the powerpc tree along with the corresponding powerpc bits from Aneesh ?
The other option would be to have it in a topic branch that I pull from you.
Cheers,
Ben.
PATCH 4/5 powerpc: mm: Only check for _PAGE_PRESENT in set_pte/pmd functions
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We want to make sure we don't use these function when updating a pte or pmd entry that have a valid hpte entry, because these functions don't invalidate them. So limit the check to _PAGE_PRESENT bit. Numafault core changes use these functions for updating _PAGE_NUMA bits. That should be ok because when _PAGE_NUMA is set we can be sure that hpte entries are not present.
--- arch/powerpc/mm/pgtable.c | 2 +- arch/powerpc/mm/pgtable_64.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
arch/powerpc/mm/pgtable.c +1 -1
@@ -174,7 +174,7 @@ void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) {#ifdef CONFIG_DEBUG_VM -WARN_ON (pte_present (*ptep)); +WARN_ON (pte_val (*ptep) & _PAGE_PRESENT);#endif /* Note: mm->context.id might not yet have been assigned as * this context might not have been activated yet when this
arch/powerpc/mm/pgtable_64.c +1 -1
@@ -687,7 +687,7 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) {#ifdef CONFIG_DEBUG_VM -WARN_ON (!pmd_none (*pmdp)); +WARN_ON (pmd_val (*pmdp) & _PAGE_PRESENT);assert_spin_locked (&mm->page_table_lock);WARN_ON (!pmd_trans_huge (pmd));#endif
Signature · 1 line
1.8.3.2
Re: [PATCH -V2 4/5] powerpc: mm: Only check for _PAGE_PRESENT in set_pte/pmd functions
Quoted 10 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
We want to make sure we don't use these function when updating a pte or pmd entry that have a valid hpte entry, because these functions don't invalidate them. So limit the check to
_PAGE_PRESENTbit. Numafault core changes use these functions for updating_PAGE_NUMAbits. That should be ok because when_PAGE_NUMAis set we can be sure that hpte entries are not present.Signed-off-by Aneesh Kumar K.V <[email protected]>
PATCH 5/5 powerpc: mm: book3s: Enable _PAGE_NUMA for book3s
In reply to Aneesh Kumar K.VFrom: "Aneesh Kumar K.V" <[email protected]>
We steal the _PAGE_COHERENCE bit and use that for indicating NUMA ptes.
--- arch/powerpc/include/asm/pgtable.h | 66 +++++++++++++++++++++++++++++++++- arch/powerpc/include/asm/pte-hash64.h | 6 ++++ arch/powerpc/platforms/Kconfig.cputype | 1 + 3 files changed, 72 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/pgtable.h +65 -1
@@ -3,6 +3,7 @@#ifdef __KERNEL__#ifndef __ASSEMBLY__ +#include <linux/mmdebug.h> #include <asm/processor.h> /* For TASK_SIZE */ #include <asm/mmu.h> #include <asm/page.h>
@@ -33,10 +34,73 @@ static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; }static inline int pte_young (pte_t pte) {return pte_val (pte) & _PAGE_ACCESSED; }static inline int pte_file (pte_t pte) {return pte_val (pte) & _PAGE_FILE; }static inline int pte_special (pte_t pte) {return pte_val (pte) & _PAGE_SPECIAL; } -static inline int pte_present (pte_t pte) {return pte_val (pte) & _PAGE_PRESENT; }static inline int pte_none (pte_t pte) {return (pte_val (pte) & ~_PTE_NONE_MASK) ==0 ; }static inline pgprot_t pte_pgprot (pte_t pte) {return __pgprot (pte_val (pte) &PAGE_PROT_BITS ); } +#ifdef CONFIG_NUMA_BALANCING + +static inline int pte_present (pte_t pte) +{ +return pte_val (pte) & (_PAGE_PRESENT | _PAGE_NUMA); +} + +#define pte_numa pte_numa +static inline int pte_numa (pte_t pte) +{ +return (pte_val (pte) & + (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA; +} + +#define pte_mknonnuma pte_mknonnuma +static inline pte_t pte_mknonnuma (pte_t pte) +{ +pte_val (pte) &= ~_PAGE_NUMA; +pte_val (pte) |= _PAGE_PRESENT | _PAGE_ACCESSED; +return pte; +} + +#define pte_mknuma pte_mknuma +static inline pte_t pte_mknuma (pte_t pte) +{ +/* +* We should not set _PAGE_NUMA on non present ptes. Also clear the +* present bit so that hash_page will return 1 and we collect this +* as numa fault. +*/ +if (pte_present (pte)) { +pte_val (pte) |= _PAGE_NUMA; +pte_val (pte) &= ~_PAGE_PRESENT; + }else +VM_BUG_ON (1 ); +return pte; +} + +#define pmd_numa pmd_numa +static inline int pmd_numa (pmd_t pmd) +{ +return pte_numa (pmd_pte (pmd)); +} + +#define pmd_mknonnuma pmd_mknonnuma +static inline pmd_t pmd_mknonnuma (pmd_t pmd) +{ +return pte_pmd (pte_mknonnuma (pmd_pte (pmd))); +} + +#define pmd_mknuma pmd_mknuma +static inline pmd_t pmd_mknuma (pmd_t pmd) +{ +return pte_pmd (pte_mknuma (pmd_pte (pmd))); +} + +# else + +static inline int pte_present (pte_t pte) +{ +return pte_val (pte) & _PAGE_PRESENT; +} +#endif /* CONFIG_NUMA_BALANCING */ +/* Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. *
arch/powerpc/include/asm/pte-hash64.h +6 -0
@@ -27,6 +27,12 @@#define _PAGE_RW 0x0200/* software: user write access allowed */ #define _PAGE_BUSY 0x0800/* software: PTE & hash are busy */ +/* +* Used for tracking numa faults +*/ +#define _PAGE_NUMA 0x00000010/* Gather numa placement stats */ + +/* No separate kernel read-only */ #define _PAGE_KERNEL_RW (_PAGE_RW | _PAGE_DIRTY)/* user access blocked by key */ #define _PAGE_KERNEL_RO _PAGE_KERNEL_RW
arch/powerpc/platforms/Kconfig.cputype +1 -0
@@ -72,6 +72,7 @@ config PPC_BOOK3S_64 select PPC_HAVE_PMU_SUPPORT select SYS_SUPPORTS_HUGETLBFS select HAVE_ARCH_TRANSPARENT_HUGEPAGE if PPC_64K_PAGES + select ARCH_SUPPORTS_NUMA_BALANCING config PPC_BOOK3E_64 bool "Embedded processors"
Signature · 1 line
1.8.3.2
Re: [PATCH -V2 5/5] powerpc: mm: book3s: Enable _PAGE_NUMA for book3s
Quoted 5 lines · Aneesh Kumar K.V
From: "Aneesh Kumar K.V" <[email protected]>
We steal the
_PAGE_COHERENCEbit and use that for indicating NUMA ptes.Signed-off-by Aneesh Kumar K.V <[email protected]>