incoming
Subsystems (109)
Andrew Morton
maintainerPATCH 124/192 mm/mremap: use vma_lookup() in vma_to_resize()
In reply to Andrew MortonFrom: Liam Howlett <[email protected]>
Subject: mm/mremap: use vma_lookup() in vma_to_resize()
Use vma_lookup() to find the VMA at a specific address. As vma_lookup() will return NULL if the address is not within any VMA, the start address no longer needs to be validated.
---
mm/mremap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
mm/mremap.c renamed from mm/mremap.c~mm-mremap-use-vma_lookup-in-vma_to_resize +3 -2
@@ -634,10 +634,11 @@ static struct vm_area_struct *vma_to_res unsigned long *p) {struct mm_struct *mm = current->mm; -struct vm_area_struct *vma =find_vma (mm, addr); +struct vm_area_struct *vma;unsigned long pgoff; -if (!vma || vma->vm_start > addr) + vma =vma_lookup (mm, addr); +if (!vma)return ERR_PTR (-EFAULT );/*
_
Andrew Morton
maintainerPATCH 125/192 mm/memory.c: use vma_lookup() in __access_remote_vm()
In reply to Andrew MortonFrom: Liam Howlett <[email protected]>
Subject: mm/memory.c: use vma_lookup() in __access_remote_vm()
Use vma_lookup() to find the VMA at a specific address. As vma_lookup() will return NULL if the address is not within any VMA, the start address no longer needs to be validated.
---
mm/memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
mm/memory.c renamed from mm/memory.c~mm-memoryc-use-vma_lookup-in-__access_remote_vm +2 -2
@@ -4994,8 +4994,8 @@ int __access_remote_vm(struct mm_struct* Check if this is a VM_IO | VM_PFNMAP VMA, which * we can access using slightly different code. */ - vma =find_vma (mm, addr); -if (!vma || vma->vm_start > addr) + vma =vma_lookup (mm, addr); +if (!vma)break ;if (vma->vm_ops && vma->vm_ops->access) ret = vma->vm_ops->access(vma, addr, buf,
_
Andrew Morton
maintainerPATCH 126/192 mm/mempolicy: use vma_lookup() in __access_remote_vm()
In reply to Andrew MortonFrom: Liam Howlett <[email protected]>
Subject: mm/mempolicy: use vma_lookup() in __access_remote_vm()
vma_lookup() finds the vma of a specific address with a cleaner interface and is more readable.
---
mm/mempolicy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/mempolicy.c renamed from mm/mempolicy.c~mm-mempolicy-use-vma_lookup-in-__access_remote_vm +1 -1
@@ -975,7 +975,7 @@ static long do_get_mempolicy(int *policy* want to return MPOL_DEFAULT in this case. */ mmap_read_lock (mm); - vma =find_vma_intersection (mm, addr, addr+1 ); + vma =vma_lookup (mm, addr);if (!vma) {mmap_read_unlock (mm);return -EFAULT ;
_
Andrew Morton
maintainerPATCH 127/192 mm: update legacy flush_tlb_* to use vma
In reply to Andrew MortonFrom: Chen Li <[email protected]>
Subject: mm: update legacy flush_tlb_* to use vma
1. These tlb flush functions have been using vma instead mm long time ago, but there is still some coments use mm as parameter.
2. the actual struct we use is vm_area_struct instead of vma_struct.
3. remove unused flush_kern_tlb_page.
---
arch/arm/include/asm/tlbflush.h | 13 +++---------- arch/arm/mm/tlb-v6.S | 2 +- arch/arm/mm/tlb-v7.S | 2 +- arch/ia64/kernel/efi_stub.S | 2 +- arch/m68k/include/asm/tlbflush.h | 2 +- arch/openrisc/include/asm/tlbflush.h | 2 +- arch/xtensa/include/asm/tlbflush.h | 4 ++-- 7 files changed, 10 insertions(+), 17 deletions(-)
arch/arm/include/asm/tlbflush.h renamed from arch/arm/include/asm/tlbflush.h~mm-update-legacy-flush_tlb_-to-use-vma +3 -10
@@ -253,7 +253,7 @@ extern struct cpu_tlb_fns cpu_tlb;* space. * - mm - mm_struct describing address space * -* flush_tlb_range(mm,start,end) +* flush_tlb_range(vma,start,end) * * Invalidate a range of TLB entries in the specified * address space.
@@ -261,18 +261,11 @@ extern struct cpu_tlb_fns cpu_tlb;* - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) * -* flush_tlb_page(vaddr,vma) +* flush_tlb_page(vma, uaddr) * * Invalidate the specified page in the specified address range. +* - vma - vm_area_struct describing address range * - vaddr - virtual address (may not be aligned) -* - vma - vma_struct describing address range -* -* flush_kern_tlb_page(kaddr) -* -* Invalidate the TLB entry for the specified page. The address -* will be in the kernels virtual memory space. Current uses -* only require the D-TLB to be invalidated. -* - kaddr - Kernel virtual memory address */ /*
arch/arm/mm/tlb-v6.S renamed from arch/arm/mm/tlb-v6.S~mm-update-legacy-flush_tlb_-to-use-vma +1 -1
@@ -24,7 +24,7 @@ * * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) - * - vma - vma_struct describing address range + * - vma - vm_area_struct describing address range * * It is assumed that: * - the "Invalidate single entry" instruction will invalidate
arch/arm/mm/tlb-v7.S renamed from arch/arm/mm/tlb-v7.S~mm-update-legacy-flush_tlb_-to-use-vma +1 -1
@@ -23,7 +23,7 @@ * * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) - * - vma - vma_struct describing address range + * - vma - vm_area_struct describing address range * * It is assumed that: * - the "Invalidate single entry" instruction will invalidate
arch/ia64/kernel/efi_stub.S renamed from arch/ia64/kernel/efi_stub.S~mm-update-legacy-flush_tlb_-to-use-vma +1 -1
@@ -7,7 +7,7 @@ * * This stub allows us to make EFI calls in physical mode with interrupts * turned off. We need this because we can't call SetVirtualMap() until - * the kernel has booted far enough to allow allocation of struct vma_struct + * the kernel has booted far enough to allow allocation of struct vm_area_struct * entries (which we would need to map stuff with memory attributes other * than uncached or writeback...). Since the GetTime() service gets called * earlier than that, we need to be able to make physical mode EFI calls from
arch/m68k/include/asm/tlbflush.h renamed from arch/m68k/include/asm/tlbflush.h~mm-update-legacy-flush_tlb_-to-use-vma +1 -1
@@ -263,7 +263,7 @@ static inline void flush_tlb_page(structBUG (); } -static inline void flush_tlb_range (struct mm_struct *mm, +static inline void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end) {BUG ();
arch/openrisc/include/asm/tlbflush.h renamed from arch/openrisc/include/asm/tlbflush.h~mm-update-legacy-flush_tlb_-to-use-vma +1 -1
@@ -25,7 +25,7 @@* - flush_tlb_all() flushes all processes TLBs * - flush_tlb_mm(mm) flushes the specified mm context TLB's * - flush_tlb_page(vma, vmaddr) flushes one page -* - flush_tlb_range(mm, start, end) flushes a range of pages +* - flush_tlb_range(vma, start, end) flushes a range of pages */ extern void local_flush_tlb_all (void );extern void local_flush_tlb_mm (struct mm_struct *mm);
arch/xtensa/include/asm/tlbflush.h renamed from arch/xtensa/include/asm/tlbflush.h~mm-update-legacy-flush_tlb_-to-use-vma +2 -2
@@ -26,8 +26,8 @@* * - flush_tlb_all() flushes all processes TLB entries * - flush_tlb_mm(mm) flushes the specified mm context TLB entries -* - flush_tlb_page(mm, vmaddr) flushes a single page -* - flush_tlb_range(mm, start, end) flushes a range of pages +* - flush_tlb_page(vma, page) flushes a single page +* - flush_tlb_range(vma, vmaddr, end) flushes a range of pages */ void local_flush_tlb_all (void );
_
Andrew Morton
maintainerPATCH 128/192 mm: improve mprotect(R|W) efficiency on pages referenced once
In reply to Andrew MortonFrom: Peter Collingbourne <[email protected]>
Subject: mm: improve mprotect(R|W) efficiency on pages referenced once
In the Scudo memory allocator [1] we would like to be able to detect use-after-free vulnerabilities involving large allocations by issuing mprotect(PROT_NONE) on the memory region used for the allocation when it is deallocated. Later on, after the memory region has been "quarantined" for a sufficient period of time we would like to be able to use it for another allocation by issuing mprotect(PROT_READ|PROT_WRITE).
Before this patch, after removing the write protection, any writes to the memory region would result in page faults and entering the copy-on-write code path, even in the usual case where the pages are only referenced by a single PTE, harming performance unnecessarily. Make it so that any pages in anonymous mappings that are only referenced by a single PTE are immediately made writable during the mprotect so that we can avoid the page faults.
This program shows the critical syscall sequence that we intend to use in the allocator:
#include <string.h>
#include <sys/mman.h>
enum { kSize = 131072 };
int main(int argc, char **argv) {
char *addr = (char *)mmap(0, kSize, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
for (int i = 0; i != 100000; ++i) {
memset(addr, i, kSize);
mprotect((void *)addr, kSize, PROT_NONE);
mprotect((void *)addr, kSize, PROT_READ | PROT_WRITE);
}
}The effect of this patch on the above program was measured on a DragonBoard 845c by taking the median real time execution time of 10 runs.
Before: 2.94s After: 0.66s
The effect was also measured using one of the microbenchmarks that we normally use to benchmark the allocator [2], after modifying it to make the appropriate mprotect calls [3]. With an allocation size of 131072 bytes to trigger the allocator's "large allocation" code path the per-iteration time was measured as follows:
Before: 27450ns
After: 6010ns
This patch means that we do more work during the mprotect call itself in exchange for less work when the pages are accessed. In the worst case, the pages are not accessed at all. The effect of this patch in such cases was measured using the following program:
#include <string.h>
#include <sys/mman.h>
enum { kSize = 131072 };
int main(int argc, char **argv) {
char *addr = (char *)mmap(0, kSize, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
memset(addr, 1, kSize);
for (int i = 0; i != 100000; ++i) {
#ifdef PAGE_FAULT
memset(addr + (i * 4096) % kSize, i, 4096);
#endif
mprotect((void *)addr, kSize, PROT_NONE);
mprotect((void *)addr, kSize, PROT_READ | PROT_WRITE);
}
}With PAGE_FAULT undefined (0 pages touched after removing write protection) the median real time execution time of 100 runs was measured as follows:
Before: 0.330260s After: 0.338836s
With PAGE_FAULT defined (1 page touched) the measurements were as follows:
Before: 0.438048s After: 0.355661s
So it seems that even with a single page fault the new approach is faster.
I saw similar results if I adjusted the programs to use a larger mapping size. With kSize = 1048576 I get these numbers with PAGE_FAULT undefined:
Before: 1.428988s After: 1.512016s
i.e. around 5.5%.
And these with PAGE_FAULT defined:
Before: 1.518559s After: 1.524417s
i.e. about the same.
What I think we may conclude from these results is that for smaller mappings the advantage of the previous approach, although measurable, is wiped out by a single page fault. I think we may expect that there should be at least one access resulting in a page fault (under the previous approach) after making the pages writable, since the program presumably made the pages writable for a reason.
For larger mappings we may guesstimate that the new approach wins if the density of future page faults is > 0.4%. But for the mappings that are large enough for density to matter (not just the absolute number of page faults) it doesn't seem like the increase in mprotect latency would be very large relative to the total mprotect execution time.
---
mm/mprotect.c | 52 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-)
mm/mprotect.c renamed from mm/mprotect.c~mm-improve-mprotectrw-efficiency-on-pages-referenced-once +46 -6
@@ -35,6 +35,51 @@#include "internal.h" +/* Determine whether we can avoid taking write faults for known dirty pages. */ +static bool may_avoid_write_fault (pte_t pte,struct vm_area_struct *vma, +unsigned long cp_flags) +{ +/* +* The dirty accountable bit indicates that we can always make the page +* writable regardless of the number of references. +*/ +if (!(cp_flags &MM_CP_DIRTY_ACCT )) { +/* Otherwise, we must have exclusive access to the page. */ +if (!(vma_is_anonymous (vma) && (vma->vm_flags &VM_WRITE ))) +return false; + +if (page_count (pte_page (pte)) !=1 ) +return false; + } + +/* +* Don't do this optimization for clean pages as we need to be notified +* of the transition from clean to dirty. +*/ +if (!pte_dirty (pte)) +return false; + +/* Same for softdirty. */ +if (!pte_soft_dirty (pte) && (vma->vm_flags &VM_SOFTDIRTY )) +return false; + +/* +* For userfaultfd the user program needs to monitor write faults so we +* can't do this optimization. +*/ +if (pte_uffd_wp (pte)) +return false; + +/* +* It is unclear whether this optimization can be done safely for NUMA +* pages. +*/ +if (cp_flags &MM_CP_PROT_NUMA ) +return false; + +return true; +} +static unsigned long change_pte_range (struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr,unsigned long end,pgprot_t newprot,unsigned long cp_flags)
@@ -43,7 +88,6 @@ static unsigned long change_pte_range(stspinlock_t *ptl;unsigned long pages =0 ;int target_node =NUMA_NO_NODE ; -bool dirty_accountable = cp_flags &MM_CP_DIRTY_ACCT ;bool prot_numa = cp_flags &MM_CP_PROT_NUMA ;bool uffd_wp = cp_flags &MM_CP_UFFD_WP ;bool uffd_wp_resolve = cp_flags &MM_CP_UFFD_WP_RESOLVE ;
@@ -131,12 +175,8 @@ static unsigned long change_pte_range(st ptent =pte_clear_uffd_wp (ptent); } -/* Avoid taking write faults for known dirty pages */ -if (dirty_accountable &&pte_dirty (ptent) && - (pte_soft_dirty (ptent) || - !(vma->vm_flags &VM_SOFTDIRTY ))) { +if (may_avoid_write_fault (ptent, vma, cp_flags)) ptent =pte_mkwrite (ptent); - }ptep_modify_prot_commit (vma, addr, pte, oldpte, ptent); pages++; }else if (is_swap_pte (oldpte)) {
_
Re: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
mm/mprotect.c static unsigned long change_pte_range(st · line 139
Quoted 9 lines · Andrew Morton
mm/mprotect.c renamed from mm/mprotect.c~mm-improve-mprotectrw-efficiency-on-pages-referenced-once +1
-5-/* Avoid taking write faults for known dirty pages */ -if (dirty_accountable &&pte_dirty (ptent) && - (pte_soft_dirty (ptent) || - !(vma->vm_flags &VM_SOFTDIRTY ))) { +if (may_avoid_write_fault (ptent, vma, cp_flags)) ptent =pte_mkwrite (ptent); - }
Hmm. I don't think this is correct.
As fat as I can tell, may_avoid_write_fault() doesn't even check if the vma is writable!
Am I misreading it? Because I think you just made even a shared mmap with "mprotect(PROT_READ)" turn the pte's writable.
Which is a "slight" security issue.
Maybe the new code is fine, and I'm missing something. The old code looks strange too, which makes me think that the MM_CP_DIRTY_ACCT test ends up saving us and depend on VM_WRITE. But it's very much not obvious.
And even if I _am_ missing something, I really would like a very obvious and direct test for "this vma is writable", ie maybe a
if (!(vma->vm_flags & VM_WRITE))
return false;at the very top of the function.
And no, "pte_dirty()" is not a reason to make something writable, it might have started out as a writable mapping, and we dirtied the page, and we made it read-only. The page stays dirty, but it shouldn't become writable just because of that.
So please make me get the warm and fuzzies about this code. Because as-is, it just looks scary.
Linus
Peter Xu
reviewerRe: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 19 lines · Linus Torvalds
mm/mprotect.c static unsigned long change_pte_range(st · line 139
Quoted 9 lines · Andrew Morton
mm/mprotect.c renamed from mm/mprotect.c~mm-improve-mprotectrw-efficiency-on-pages-referenced-once +1
-5-/* Avoid taking write faults for known dirty pages */ -if (dirty_accountable &&pte_dirty (ptent) && - (pte_soft_dirty (ptent) || - !(vma->vm_flags &VM_SOFTDIRTY ))) { +if (may_avoid_write_fault (ptent, vma, cp_flags)) ptent =pte_mkwrite (ptent); - }Hmm. I don't think this is correct.
As fat as I can tell,
may_avoid_write_fault()doesn't even check if the vma is writable!Am I misreading it? Because I think you just made even a shared mmap with "
mprotect(PROT_READ)" turn the pte's writable.
Quoted 6 lines
Which is a "slight" security issue.
Maybe the new code is fine, and I'm missing something. The old code looks strange too, which makes me think that the
MM_CP_DIRTY_ACCTtest ends up saving us and depend onVM_WRITE. But it's very much not obvious.
vma_wants_writenotify() checks first VM_WRITE|VM_SHARED, otherwise MM_CP_DIRTY_ACCT will not be set. While for anonymous vmas the newly introduced may_avoid_write_fault() checks VM_WRITE explicitly.
Agreed even if it's checked it's not straightforward. Maybe it'll be a bonus to have a comment above may_avoid_write_fault() about it in a follow up.
Quoted 8 lines
And even if I _am_ missing something, I really would like a very obvious and direct test for "this vma is writable", ie maybe a
if (!(vma->vm_flags & VM_WRITE)) return false;at the very top of the function.
Yes looks okay too; I think using MM_CP_DIRTY_ACCT flag has a slight advantage in that it checks VM_WRITE only once before calling change_protection(), rather than doing the check for every pte even if we know it'll have the same result. However it indeed hides the facts deeper..
Quoted 5 lines
And no, "
pte_dirty()" is not a reason to make something writable, it might have started out as a writable mapping, and we dirtied the page, and we made it read-only. The page stays dirty, but it shouldn't become writable just because of that.
I think the dirty bit checks are only to make sure we don't need those extra write faults. It should definitely be based on the fact that VM_WRITE being set already.
Thanks,
Signature · 1 line
Peter Xu
Peter Xu
reviewerRe: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 22 lines · Peter Xu
Quoted 21 lines · Linus Torvalds
mm/mprotect.c static unsigned long change_pte_range(st · line 139
Quoted 9 lines · Andrew Morton
mm/mprotect.c renamed from mm/mprotect.c~mm-improve-mprotectrw-efficiency-on-pages-referenced-once +1
-5-/* Avoid taking write faults for known dirty pages */ -if (dirty_accountable &&pte_dirty (ptent) && - (pte_soft_dirty (ptent) || - !(vma->vm_flags &VM_SOFTDIRTY ))) { +if (may_avoid_write_fault (ptent, vma, cp_flags)) ptent =pte_mkwrite (ptent); - }Hmm. I don't think this is correct.
As fat as I can tell,
may_avoid_write_fault()doesn't even check if the vma is writable!Am I misreading it? Because I think you just made even a shared mmap with "
mprotect(PROT_READ)" turn the pte's writable.Which is a "slight" security issue.
Quoted 8 lines
Quoted 4 lines
Maybe the new code is fine, and I'm missing something. The old code looks strange too, which makes me think that the
MM_CP_DIRTY_ACCTtest ends up saving us and depend onVM_WRITE. But it's very much not obvious.
vma_wants_writenotify()checks first VM_WRITE|VM_SHARED, otherwiseMM_CP_DIRTY_ACCTwill not be set. While for anonymous vmas the newly introducedmay_avoid_write_fault()checksVM_WRITEexplicitly.
Sorry, this statement is unclear. It's not about anonymous or not, it's just that a hidden check against VM_WRITE is there already..
Say, below chunk of the patch:
if (!(cp_flags & MM_CP_DIRTY_ACCT)) {
/* Otherwise, we must have exclusive access to the page. */ if (!(vma_is_anonymous(vma) && (vma->vm_flags & VM_WRITE)))
return false;
if (page_count(pte_page(pte)) != 1)
return false;
}
Should be the same as:
if (!(cp_flags & MM_CP_DIRTY_ACCT)) {
if (!vma_is_anonymous(vma))
return false;
if (!(vma->vm_flags & VM_WRITE))
return false; if (page_count(pte_page(pte)) != 1)
return false;
}
And since MM_CP_DIRTY_ACCT implies "VM_WRITE|VM_SHARED" all set, above should be a slightly faster version of below:
/* This just never trigger if MM_CP_DIRTY_ACCT set */
if (!(vma->vm_flags & VM_WRITE))
return false; if (!(cp_flags & MM_CP_DIRTY_ACCT)) {
if (!vma_is_anonymous(vma))
return false;
if (page_count(pte_page(pte)) != 1)
return false;
}
It's just that we avoid checking "vma->vm_flags & VM_WRITE" when MM_CP_DIRTY_ACCT set.
Again, I think in all cases some more comment should be good indeed..
Quoted 32 lines
Agreed even if it's checked it's not straightforward. Maybe it'll be a bonus to have a comment above
may_avoid_write_fault()about it in a follow up.Quoted 8 lines
And even if I _am_ missing something, I really would like a very obvious and direct test for "this vma is writable", ie maybe a
if (!(vma->vm_flags & VM_WRITE)) return false;at the very top of the function.
Yes looks okay too; I think using
MM_CP_DIRTY_ACCTflag has a slight advantage in that it checksVM_WRITEonly once before callingchange_protection(), rather than doing the check for every pte even if we know it'll have the same result. However it indeed hides the facts deeper..Quoted 5 lines
And no, "
pte_dirty()" is not a reason to make something writable, it might have started out as a writable mapping, and we dirtied the page, and we made it read-only. The page stays dirty, but it shouldn't become writable just because of that.I think the dirty bit checks are only to make sure we don't need those extra write faults. It should definitely be based on the fact that
VM_WRITEbeing set already.Thanks,
Signature · 1 line
Peter Xu
Signature · 1 line
Peter Xu
Re: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 3 lines · Peter Xu
And since
MM_CP_DIRTY_ACCTimplies "VM_WRITE|VM_SHARED" all set, above should be a slightly faster version of below:
That's way too subtle, particularly since the MM_CP_DIRTY_ACCT logic comes from another file entirely.
I don't think it's even faster, considering that presumably the anonymous mapping case is the common one, and that's the one that needs all the extra tests, it's likely better to _not_ test that very subtle flag at all, and just doing the straightforward and obvious tests that are understandable _locally_.
So I claim that it's
(a) not an optimization at all
(b) completely locally unintuitive and unreadable
Quoted 1 line
Again, I think in all cases some more comment should be good indeed..
I really want more than a comment. I want that MM_CP_DIRTY_ACCT bit testing gone.
The only point where it makes sense to check MM_CP_DIRTY_ACCT is within the context of "is the page already dirty".
So I think the logic should be something along the lines of
- first:
if (!(vma->vm_flags & VM_WRITE))
return false; because that logic is set in stone, and true regardless of anything
else. If the vma isn't writable, we're not going to set the write bit. End of story.
- then, check the vma_is_anonumous() case:
if (vma_is_anonymous(vma))
return page_count(pte_page(pte)) == 1;
because if it's a writable mapping, and anonymous, then we can
mark it writable if we're the exclusive owners of that page.
- and THEN we can handle the "ok, shared mapping, now let's start
thinking about dirty accounting" cases.
Make it obvious and correct. This is not a sequence where you should try to (incorrectly) optimize away individual instructions.
Linus
Peter Xu
reviewerRe: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 16 lines · Linus Torvalds
Quoted 3 lines · Peter Xu
And since
MM_CP_DIRTY_ACCTimplies "VM_WRITE|VM_SHARED" all set, above should be a slightly faster version of below:That's way too subtle, particularly since the
MM_CP_DIRTY_ACCTlogic comes from another file entirely.I don't think it's even faster, considering that presumably the anonymous mapping case is the common one, and that's the one that needs all the extra tests, it's likely better to _not_ test that very subtle flag at all, and just doing the straightforward and obvious tests that are understandable _locally_.
So I claim that it's
Quoted 8 lines
(a) not an optimization at all
(b) completely locally unintuitive and unreadable
Quoted 1 line
Again, I think in all cases some more comment should be good indeed..
I really want more than a comment. I want that
MM_CP_DIRTY_ACCTbit testing gone.
My understanding is that MM_CP_DIRTY_ACCT contains all check results from vma_wants_writenotify(), so if we drop it we'd need to have something like that to be checked within change_pte_range(), which is again slower (I have totally no idea how slow to check vma->vm_flags & VM_WRITE, but moving the whole vma_wants_writenotify here is definitely even slower).
Quoted 15 lines
The only point where it makes sense to check
MM_CP_DIRTY_ACCTis within the context of "is the page already dirty".So I think the logic should be something along the lines of
- first:
if (!(vma->vm_flags & VM_WRITE)) return false;because that logic is set in stone, and true regardless of anything
else. If the vma isn't writable, we're not going to set the write bit. End of story.
Quoted 7 lines
- then, check the
vma_is_anonumous()case:if (
vma_is_anonymous(vma))
return page_count(pte_page(pte)) == 1;because if it's a writable mapping, and anonymous, then we can
mark it writable if we're the exclusive owners of that page.
Shouldn't we still at least checks [soft-]dirty bits and uffd-wp bits to make sure it's either not dirty tracked or uffd wr-protected? Say, IMHO it's possible that soft-dirty tracking enabled on this anonymous vma range, then we still depend on the write bit removed to set the soft-dirty later in the fault handler.
Quoted 6 lines
- and THEN we can handle the "ok, shared mapping, now let's start
thinking about dirty accounting" cases.Make it obvious and correct. This is not a sequence where you should try to (incorrectly) optimize away individual instructions.
Yes I still fully agree it's very un-obvious. So far the best thing I can come up with is something like below (patch attached too but not yet tested). I moved VM_WRITE out so hopefully it'll be very clear; then I also rearranged the checks so the final outcome looks like below:
static bool may_avoid_write_fault(pte_t pte, struct vm_area_struct *vma,
unsigned long cp_flags)
{
/*
* It is unclear whether this optimization can be done safely for NUMA
* pages.
*/
if (cp_flags & MM_CP_PROT_NUMA)
return false;
/*
* Never apply write bit if VM_WRITE not set. Note that this is
* actually checked for VM_SHARED when MM_CP_DIRTY_ACCT is set, so
* logically we only need to check it for !MM_CP_DIRTY_ACCT, but just
* make it even more obvious.
*/
if (!(vma->vm_flags & VM_WRITE))
return false;
/*
* Don't do this optimization for clean pages as we need to be notified
* of the transition from clean to dirty.
*/
if (!pte_dirty(pte))
return false;
/* Same for softdirty. */
if (!pte_soft_dirty(pte) && (vma->vm_flags & VM_SOFTDIRTY))
return false;
/*
* For userfaultfd the user program needs to monitor write faults so we
* can't do this optimization.
*/
if (pte_uffd_wp(pte))
return false;
/*
* MM_CP_DIRTY_ACCT indicates that we can always make the page writable
* regardless of the number of references. Time to set the write bit.
*/
if (cp_flags & MM_CP_DIRTY_ACCT)
return true;
/*
* Othewise it means !MM_CP_DIRTY_ACCT. We can only apply write bit
* early if it's anonymous page and we exclusively own it.
*/
if (vma_is_anonymous(vma) && (page_count(pte_page(pte)) == 1))
return true;
/* Don't play any trick */
return false;
}The logic should be the same as before, it's just that we'll do an extra check on VM_WRITE for MM_CP_DIRTY_ACCT but assuming it's ok.
Another side note is that I still think the VM_SOFTDIRTY check is wrong in may_avoid_write_fault() and even in the old code (I mentioned it previously when reviewing the patch), as !VM_SOFTDIRTY should mean soft dirty tracking enabled while VM_SOFTDIRTY means disabled. So I wonder whether it should be:
- if (!pte_soft_dirty(pte) && (vma->vm_flags & VM_SOFTDIRTY))
+ if (!pte_soft_dirty(pte) && !(vma->vm_flags & VM_SOFTDIRTY))
However I didn't touch it up there as it may need more justifications (I feel it's okay in the old code, as vma_wants_writenotify actually checks it too and in the right way; however after the anonymous fast path it seems to prone to error if it's anonymous; I'll check later).
Thanks,
Signature · 1 line
Peter Xu
From 4fb32ad7c949d5ec6b6ea364d3388b50bf674c9c Mon Sep 17 00:00:00 2001
From: Peter Xu <[email protected]>
Date: Wed, 30 Jun 2021 12:20:12 -0400
Subject: [PATCH] mm/mprotect: Optimize layout of may_avoid_write_fault()
Firstly move VM_WRITE check to be outside of !MM_CP_DIRTY_ACCT chunk, so as to make it clear that we won't accidentally set the write bit to !VM_WRITE vmas.
The old logic is hard to read in that it was written in reversed logic. Put things backward by moving the soft-dirty and uffd-wp checks earlier. Make the NUMA check even earlier than those as it's a cheap check and straightforward.
Make the only "return true" case to be either the MM_CP_DIRTY_ACCT (which stands for the VM_SHARED cases when write bit can be applied), or the special anonymous page when we exclusively own it.
--- mm/mprotect.c | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-)
mm/mprotect.c +25 -14
@@ -40,17 +40,20 @@ static bool may_avoid_write_fault(pte_t pte, struct vm_area_struct *vma, unsigned long cp_flags) {/* -* The dirty accountable bit indicates that we can always make the page -* writable regardless of the number of references. +* It is unclear whether this optimization can be done safely for NUMA +* pages. */ -if (!(cp_flags &MM_CP_DIRTY_ACCT )) { -/* Otherwise, we must have exclusive access to the page. */ -if (!(vma_is_anonymous (vma) && (vma->vm_flags &VM_WRITE ))) -return false; +if (cp_flags &MM_CP_PROT_NUMA ) +return false; -if (page_count (pte_page (pte)) !=1 ) -return false; - } +/* +* Never apply write bit if VM_WRITE not set. Note that this is +* actually checked for VM_SHARED when MM_CP_DIRTY_ACCT is set, so +* logically we only need to check it for !MM_CP_DIRTY_ACCT, but just +* make it even more obvious. +*/ +if (!(vma->vm_flags &VM_WRITE )) +return false;/* * Don't do this optimization for clean pages as we need to be notified
@@ -71,13 +74,21 @@ static bool may_avoid_write_fault(pte_t pte, struct vm_area_struct *vma,return false;/* -* It is unclear whether this optimization can be done safely for NUMA -* pages. +* MM_CP_DIRTY_ACCT indicates that we can always make the page writable +* regardless of the number of references. Time to set the write bit. */ -if (cp_flags &MM_CP_PROT_NUMA ) -return false; +if (cp_flags &MM_CP_DIRTY_ACCT ) +return true; + +/* +* Othewise it means !MM_CP_DIRTY_ACCT. We can only apply write bit +* early if it's anonymous page and we exclusively own it. +*/ +if (vma_is_anonymous (vma) && (page_count (pte_page (pte)) ==1 )) +return true; -return true; +/* Don't play any trick */ +return false; } static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
Signature · 1 line
2.31.1
Re: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 6 lines · Peter Xu
Yes I still fully agree it's very un-obvious. So far the best thing I can come up with is something like below (patch attached too but not yet tested). I moved
VM_WRITEout so hopefully it'll be very clear; then I also rearranged the checks so the final outcome looks like below:
Quoted 9 lines
static bool may_avoid_write_fault(pte_t pte, struct
vm_area_struct*vma,
unsigned longcp_flags)
{
/*
* It is unclear whether this optimization can be done safely for NUMA
* pages.
*/
if (cp_flags&MM_CP_PROT_NUMA)
return false;
Please just put that VM_WRITE test first. It's the one that really *matters*. There's no "it's unclear if" about that part. Just handle the obvious and important check first.
Yeah, yeah, they both return false, so order doesn't matter from a semantic standpoint, but from a clarity standpoint just do the clear and unambiguous and security-relevant test first.
The rest of the tests are implementation details, the VM_WRITE test is fundamental behavior. It's the one that made me worry about this patch in the first place.
Quoted 9 lines
/*
* Don't do this optimization for clean pages as we need to be notified
* of the transition from clean to dirty.
*/
if (!pte_dirty(pte))
return false;
/* Same for softdirty. */
if (!pte_soft_dirty(pte) && (vma->vm_flags & VM_SOFTDIRTY))Quoted 8 lines
return false;
/*
* For userfaultfd the user program needs to monitor write faults so we
* can't do this optimization.
*/
if (pte_uffd_wp(pte))
return false;So all of these are a bit special.
Why? Because if I look at the actual page fault path, these are not the tests there.
I'd really like to have some obvious situation where we keep this "make it writable" in sync with what would actually happen on a write fault when it's not writable.
And it's not at all obvious to me for these cases.
The do_wp_page() code doesn't even use pte_uffd_wp(). It uses userfaultfd_pte_wp(vma, pte), and I don't even know why. Yes, I can see the code (it additionally tests the VM_UFFD_WP flag in the vma), but a number of other paths then only do that pte_uffd_wp() test.
I get the feeling that we really should try to match what the do_wp_page() path does, though.
Which brings up another issue: the do_wp_page() path treats PageKsm() pages differently. And it locks the page before double-checking the page count.
Why does mprotect() not need to do the same thing? I think this has come up before, and "change_protection()" can get called with the mmap_sem held just for reading - see userfaultfd - so it has all the same issues as a page fault does, afaik.
Quoted 18 lines
/*
* MM_CP_DIRTY_ACCT indicates that we can always make the page writable
* regardless of the number of references. Time to set the write bit.
*/
if (cp_flags & MM_CP_DIRTY_ACCT)
return true;
/*
* Othewise it means !MM_CP_DIRTY_ACCT. We can only apply write bit
* early if it's anonymous page and we exclusively own it.
*/
if (vma_is_anonymous(vma) && (page_count(pte_page(pte)) == 1))
return true;
/* Don't play any trick */
return false;
}Quoted 2 lines
The logic should be the same as before, it's just that we'll do an extra check on
VM_WRITEforMM_CP_DIRTY_ACCTbut assuming it's ok.
See above. I don't think the logic before was all that clear either.
The one case that is clear is that if it's a shared mapping, and MM_CP_DIRTY_ACCT is set, and it was already dirty (and softdirty), then it's ok.,
That's the old code. I don't like how the old code was written (because I think that MM_CP_DIRTY_ACCT bit wasx too subtle), but I think the old code was at least correct.
The new code, it just worries me. It adds all those new cases for when we can make the page writable early - that's the whole point of the patch, after all - but my point here is that it's not at all obvious that those new cases are actually correct.
MAYBE it's all correct. I'm not saying it's wrong. I'm just saying it's not _obvious_ that it's correct.
What about that page_count() test, for example: it has a comment, it looks obvious, but it's very different from what do_wp_page() does. So what happens if we have a page-out at the same time that turns that page into a swap cache page, and increments the page count? What about that race? Do we end up with a writable page that is shared with a swap cache entry? Is that ok? Why isn't it ok in the page fault case?
See why this patch worries me so much?
Linus
Peter Xu
reviewerRe: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 21 lines · Linus Torvalds
Quoted 6 lines · Peter Xu
Yes I still fully agree it's very un-obvious. So far the best thing I can come up with is something like below (patch attached too but not yet tested). I moved
VM_WRITEout so hopefully it'll be very clear; then I also rearranged the checks so the final outcome looks like below:Quoted 9 lines
static bool may_avoid_write_fault(pte_t pte, struct
vm_area_struct*vma,
unsigned longcp_flags)
{
/*
* It is unclear whether this optimization can be done safely for NUMA
* pages.
*/
if (cp_flags&MM_CP_PROT_NUMA)
return false;Please just put that
VM_WRITEtest first. It's the one that really *matters*. There's no "it's unclear if" about that part. Just handle the obvious and important check first.
Quoted 7 lines
Yeah, yeah, they both return false, so order doesn't matter from a semantic standpoint, but from a clarity standpoint just do the clear and unambiguous and security-relevant test first.
The rest of the tests are implementation details, the
VM_WRITEtest is fundamental behavior. It's the one that made me worry about this patch in the first place.
Sure.
Quoted 28 lines
Quoted 10 lines
/* * Don't do this optimization for clean pages as we need to be notified * of the transition from clean to dirty. */ if (!pte_dirty(pte)) return false; /* Same for softdirty. */ if (!pte_soft_dirty(pte) && (vma->vm_flags & VM_SOFTDIRTY))Quoted 8 lines
return false; /* * For userfaultfd the user program needs to monitor write faults so we * can't do this optimization. */ if (pte_uffd_wp(pte)) return false;So all of these are a bit special.
Why? Because if I look at the actual page fault path, these are not the tests there.
I'd really like to have some obvious situation where we keep this "make it writable" in sync with what would actually happen on a write fault when it's not writable.
Quoted 6 lines
And it's not at all obvious to me for these cases.
The
do_wp_page()code doesn't even usepte_uffd_wp(). It usesuserfaultfd_pte_wp(vma, pte), and I don't even know why. Yes, I can see the code (it additionally tests theVM_UFFD_WPflag in the vma), but a number of other paths then only do thatpte_uffd_wp()test.
The vma check is a safety net to make sure it's not the case e.g. when the vma has already unregistered from uffd-wp while there's uffd-wp bit left over. E.g., currently UFFDIO_UNREGISTER is lazy on removing uffd-wp bits that applied to ptes, so even vma is unregistered there could still have pte_uffd_wp() being true for some ptes. That vma check makes sure when it happens the uffd-wp bit will be auto-removed.
Quoted 3 lines
I get the feeling that we really should try to match what the
do_wp_page()path does, though.
Makes sense.
Quoted 9 lines
Which brings up another issue: the
do_wp_page()path treatsPageKsm()pages differently. And it locks the page before double-checking the page count.Why does
mprotect()not need to do the same thing? I think this has come up before, and "change_protection()" can get called with themmap_semheld just for reading - see userfaultfd - so it has all the same issues as a page fault does, afaik.
Good point.. I overlooked ksm when reviewing the patch, while I should really have looked at do_wp_page() as you suggested (hmm.. the truth is I wasn't even aware of this patch and never planned to try to review it, until it breaks the uffd-wp anonymous tests in its initial versions when I was testing mmots...).
Maybe something like this (to be squashed into the previously attached patch):
---8<---
mm/mprotect.c +23 -13
@@ -39,12 +39,8 @@static bool may_avoid_write_fault (pte_t pte,struct vm_area_struct *vma,unsigned long cp_flags) { -/* -* It is unclear whether this optimization can be done safely for NUMA -* pages. -*/ -if (cp_flags &MM_CP_PROT_NUMA ) -return false; +struct page *page; +bool ret = false;/* * Never apply write bit if VM_WRITE not set. Note that this is
@@ -55,6 +51,13 @@ static bool may_avoid_write_fault(pte_t pte, struct vm_area_struct *vma,if (!(vma->vm_flags &VM_WRITE ))return false; +/* +* It is unclear whether this optimization can be done safely for NUMA +* pages. +*/ +if (cp_flags &MM_CP_PROT_NUMA ) +return false; +/* * Don't do this optimization for clean pages as we need to be notified * of the transition from clean to dirty.
@@ -80,15 +83,22 @@ static bool may_avoid_write_fault(pte_t pte, struct vm_area_struct *vma,if (cp_flags &MM_CP_DIRTY_ACCT )return true; + page =pte_page (pte); +/* Best effort to take page lock, don't play trick if failed */ +if (!trylock_page (page)) +return false; +/* KSM pages needs COW; leave them be */ +if (PageKsm (page)) + goto unlock_fail;/* -* Othewise it means !MM_CP_DIRTY_ACCT. We can only apply write bit -* early if it's anonymous page and we exclusively own it. +* Othewise it means !MM_CP_DIRTY_ACCT and !KSM. We can only apply +* write bit early if it's anonymous page and we exclusively own it. */ -if (vma_is_anonymous (vma) && (page_count (pte_page (pte)) ==1 )) -return true; - -/* Don't play any trick */ -return false; +if (vma_is_anonymous (vma) && (page_count (page) ==1 )) + ret = true; +unlock_fail: +unlock_page (page); +return ret; } static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
---8<---
I hope I didn't overlook something else..
Today when I was looking at ksm code, I found that I got lost on why we say "PageKsm() doesn't necessarily raise the page refcount", as in do_wp_page(). I was looking at replace_page() where, afaict, we still do proper refcounting for the stable nodes with "get_page(kpage)".
I know I must missed something but I can't quickly tell. In all cases with above PageKsm check I think it'll be safe, and it's definitely clear that page lock will stablize PageKsm()'s return value, like do_wp_page().
Quoted 28 lines
Quoted 19 lines
/* * MM_CP_DIRTY_ACCT indicates that we can always make the page writable * regardless of the number of references. Time to set the write bit. */ if (cp_flags & MM_CP_DIRTY_ACCT) return true; /* * Othewise it means !MM_CP_DIRTY_ACCT. We can only apply write bit * early if it's anonymous page and we exclusively own it. */ if (vma_is_anonymous(vma) && (page_count(pte_page(pte)) == 1)) return true; /* Don't play any trick */ return false; }Quoted 2 lines
The logic should be the same as before, it's just that we'll do an extra check on
VM_WRITEforMM_CP_DIRTY_ACCTbut assuming it's ok.See above. I don't think the logic before was all that clear either.
The one case that is clear is that if it's a shared mapping, and
MM_CP_DIRTY_ACCTis set, and it was already dirty (and softdirty), then it's ok.,
Quoted 8 lines
That's the old code. I don't like how the old code was written (because I think that
MM_CP_DIRTY_ACCTbit wasx too subtle), but I think the old code was at least correct.The new code, it just worries me. It adds all those new cases for when we can make the page writable early - that's the whole point of the patch, after all - but my point here is that it's not at all obvious that those new cases are actually correct.
Yes agreed the MM_CP_DIRTY_ACCT bit is very subtle and not easy to get. It's just that I don't have a good idea to make it better, yet..
Quoted 10 lines
MAYBE it's all correct. I'm not saying it's wrong. I'm just saying it's not _obvious_ that it's correct.
What about that
page_count()test, for example: it has a comment, it looks obvious, but it's very different from whatdo_wp_page()does. So what happens if we have a page-out at the same time that turns that page into a swap cache page, and increments the page count? What about that race? Do we end up with a writable page that is shared with a swap cache entry? Is that ok? Why isn't it ok in the page fault case?
That looks fine to me: when the race happens we must have checked page_count==1 first and granted the write bit, then add_to_swap_cache() happens after the page_count==1 check (as it takes another refcount, so >2 otherwise). Then it also means unmap mappings should happen even after that point. If my above understanding is correct, our newly installed pte will be zapped safely soon, but of course after we release the pgtable lock in change_pte_range().
Thanks,
Signature · 1 line
Peter Xu
Re: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
Quoted 10 lines · Peter Xu
Quoted 7 lines · Linus Torvalds
What about that
page_count()test, for example: it has a comment, it looks obvious, but it's very different from whatdo_wp_page()does. So what happens if we have a page-out at the same time that turns that page into a swap cache page, and increments the page count? What about that race? Do we end up with a writable page that is shared with a swap cache entry? Is that ok? Why isn't it ok in the page fault case?
Quoted 6 lines
That looks fine to me: when the race happens we must have checked page_count==1 first and granted the write bit, then
add_to_swap_cache()happens after the page_count==1 check (as it takes another refcount, so >2 otherwise). Then it also means unmap mappings should happen even after that point. If my above understanding is correct, our newly installed pte will be zapped safely soon, but of course after we release the pgtable lock inchange_pte_range().
So if this is fine, then maybe we should just remove the page lock in the do_wp_page() path (and remove the PageKSM check while at it)?
If it's not required by mprotect() to say "I can make the page writable directly", then it really shouldn't be required by the page fault path either.
Which I'd love to do, and was really itching to do (it's a nasty lock), but I worried about it..
I'd hate to have mprotect do one thing, and page faulting do another thing, and not have some logic to why they have to be different.
Linus
Peter Xu
reviewerRe: [patch 128/192] mm: improve mprotect(R|W) efficiency on pages referenced once
(sorry for a very late reply)
Quoted 18 lines · Linus Torvalds
Quoted 10 lines · Peter Xu
Quoted 7 lines · Linus Torvalds
What about that
page_count()test, for example: it has a comment, it looks obvious, but it's very different from whatdo_wp_page()does. So what happens if we have a page-out at the same time that turns that page into a swap cache page, and increments the page count? What about that race? Do we end up with a writable page that is shared with a swap cache entry? Is that ok? Why isn't it ok in the page fault case?Quoted 6 lines
That looks fine to me: when the race happens we must have checked page_count==1 first and granted the write bit, then
add_to_swap_cache()happens after the page_count==1 check (as it takes another refcount, so >2 otherwise). Then it also means unmap mappings should happen even after that point. If my above understanding is correct, our newly installed pte will be zapped safely soon, but of course after we release the pgtable lock inchange_pte_range().
Quoted 2 lines
So if this is fine, then maybe we should just remove the page lock in the
do_wp_page()path (and remove the PageKSM check while at it)?
I could be wrong, but I thought the page lock in do_wp_page() is more for the PageKsm() race (e.g., to make sure we don't grant write to a page that is becoming a ksm page in parallel).
Quoted 10 lines
If it's not required by
mprotect()to say "I can make the page writable directly", then it really shouldn't be required by the page fault path either.Which I'd love to do, and was really itching to do (it's a nasty lock), but I worried about it..
I'd hate to have mprotect do one thing, and page faulting do another thing, and not have some logic to why they have to be different.
Agreed; perhaps no need to be identical - I think the mprotect path can be even stricter than the fault page, as it's a fast-path only. It should never apply the write bit when the page fault path won't. So I think the original patch does need a justification on why it didn't handle ksm page while do_wp_page handled it.
Thanks,
Signature · 1 line
Peter Xu
Andrew Morton
maintainerPATCH 129/192 h8300: remove unused variable
In reply to Andrew MortonFrom: Souptick Joarder <[email protected]>
Subject: h8300: remove unused variable
Kernel test robot throws below warning ->
Quoted 1 line
Quoted 1 line
arch/h8300/kernel/setup.c:72:26:
warning: Unused variable: region [unusedVariable]
struct memblock_region *region;
Fixed it by removing unused variable.
---
arch/h8300/kernel/setup.c | 2 -- 1 file changed, 2 deletions(-)
arch/h8300/kernel/setup.c renamed from arch/h8300/kernel/setup.c~h8300-remove-unused-variable +0 -2
@@ -69,8 +69,6 @@ void __init h8300_fdt_init(void *fdt, chstatic void __init bootmem_init (void ) { -struct memblock_region *region; - memory_end = memory_start =0 ;/* Find main memory where is the kernel */
_
Andrew Morton
maintainerPATCH 130/192 mm/dmapool: use DEVICE_ATTR_RO macro
In reply to Andrew MortonFrom: YueHaibing <[email protected]>
Subject: mm/dmapool: use DEVICE_ATTR_RO macro
Use DEVICE_ATTR_RO() helper instead of plain DEVICE_ATTR(), which makes the code a bit shorter and easier to read.
---
mm/dmapool.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
mm/dmapool.c renamed from mm/dmapool.c~mm-dmapool-use-device_attr_ro-macro +2 -3
@@ -62,8 +62,7 @@ struct dma_page { /* cacheable header fstatic DEFINE_MUTEX (pools_lock);static DEFINE_MUTEX (pools_reg_lock); -static ssize_t -show_pools (struct device *dev,struct device_attribute *attr,char *buf) +static ssize_t pools_show (struct device *dev,struct device_attribute *attr,char *buf) {unsigned temp;unsigned size;
@@ -103,7 +102,7 @@ show_pools(struct device *dev, struct dereturn PAGE_SIZE - size; } -static DEVICE_ATTR(pools, 0444, show_pools, NULL); +static DEVICE_ATTR_RO (pools);/** * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
_
Andrew Morton
maintainerPATCH 131/192 mm, tracing: unify PFN format strings
In reply to Andrew MortonFrom: Vincent Whitchurch <[email protected]>
Subject: mm, tracing: unify PFN format strings
Some trace event formats print PFNs as hex while others print them as decimal. This is rather annoying when attempting to grep through traces to understand what's going on with a particular page.
$ git grep -ho 'pfn=[0x%lu]\+' include/trace/events/ | sort | uniq -c
11 pfn=0x%lx
12 pfn=%lu
2 pfn=%lxPrinting as hex is in the majority in the trace events, and all the normal printks in mm/ also print PFNs as hex, so change all the PFN formats in the trace events to use 0x%lx.
---
include/trace/events/cma.h | 4 ++-- include/trace/events/filemap.h | 2 +- include/trace/events/kmem.h | 12 ++++++------ include/trace/events/page_pool.h | 4 ++-- include/trace/events/pagemap.h | 4 ++-- include/trace/events/vmscan.h | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-)
include/trace/events/cma.h renamed from include/trace/events/cma.h~mm-tracing-unify-pfn-format-strings +2 -2
@@ -31,7 +31,7 @@ DECLARE_EVENT_CLASS(cma_alloc_class, __entry->align = align; ), - TP_printk("name=%s pfn=%lx page=%p count=%lu align=%u", + TP_printk("name=%s pfn=0x%lx page=%p count=%lu align=%u", __get_str(name), __entry->pfn, __entry->page,
@@ -60,7 +60,7 @@ TRACE_EVENT(cma_release, __entry->count = count; ), - TP_printk("name=%s pfn=%lx page=%p count=%lu", + TP_printk("name=%s pfn=0x%lx page=%p count=%lu", __get_str(name), __entry->pfn, __entry->page,
include/trace/events/filemap.h renamed from include/trace/events/filemap.h~mm-tracing-unify-pfn-format-strings +1 -1
@@ -36,7 +36,7 @@ DECLARE_EVENT_CLASS(mm_filemap_op_page_c __entry->s_dev = page->mapping->host->i_rdev; ), - TP_printk("dev %d:%d ino %lx page=%p pfn=%lu ofs=%lu", + TP_printk("dev %d:%d ino %lx page=%p pfn=0x%lx ofs=%lu", MAJOR(__entry->s_dev), MINOR(__entry->s_dev), __entry->i_ino, pfn_to_page(__entry->pfn),
include/trace/events/kmem.h renamed from include/trace/events/kmem.h~mm-tracing-unify-pfn-format-strings +6 -6
@@ -173,7 +173,7 @@ TRACE_EVENT(mm_page_free, __entry->order = order; ), -TP_printk ("page=%p pfn=%lu order=%d" , +TP_printk ("page=%p pfn=0x%lx order=%d" ,pfn_to_page (__entry->pfn), __entry->pfn, __entry->order)
@@ -193,7 +193,7 @@ TRACE_EVENT(mm_page_free_batched, __entry->pfn =page_to_pfn (page); ), -TP_printk ("page=%p pfn=%lu order=0" , +TP_printk ("page=%p pfn=0x%lx order=0" ,pfn_to_page (__entry->pfn), __entry->pfn) );
@@ -219,7 +219,7 @@ TRACE_EVENT(mm_page_alloc, __entry->migratetype = migratetype; ), - TP_printk("page=%p pfn=%lu order=%d migratetype=%d gfp_flags=%s", + TP_printk("page=%p pfn=0x%lx order=%d migratetype=%d gfp_flags=%s", __entry->pfn != -1UL ? pfn_to_page(__entry->pfn) : NULL, __entry->pfn != -1UL ? __entry->pfn : 0, __entry->order,
@@ -245,7 +245,7 @@ DECLARE_EVENT_CLASS(mm_page, __entry->migratetype = migratetype; ), - TP_printk("page=%p pfn=%lu order=%u migratetype=%d percpu_refill=%d", + TP_printk("page=%p pfn=0x%lx order=%u migratetype=%d percpu_refill=%d", __entry->pfn != -1UL ? pfn_to_page(__entry->pfn) : NULL, __entry->pfn != -1UL ? __entry->pfn : 0, __entry->order,
@@ -278,7 +278,7 @@ TRACE_EVENT(mm_page_pcpu_drain, __entry->migratetype = migratetype; ), -TP_printk ("page=%p pfn=%lu order=%d migratetype=%d" , +TP_printk ("page=%p pfn=0x%lx order=%d migratetype=%d" ,pfn_to_page (__entry->pfn), __entry->pfn, __entry->order, __entry->migratetype) );
@@ -312,7 +312,7 @@ TRACE_EVENT(mm_page_alloc_extfrag, get_pageblock_migratetype(page)); ), - TP_printk("page=%p pfn=%lu alloc_order=%d fallback_order=%d pageblock_order=%d alloc_migratetype=%d fallback_migratetype=%d fragmenting=%d change_ownership=%d", + TP_printk("page=%p pfn=0x%lx alloc_order=%d fallback_order=%d pageblock_order=%d alloc_migratetype=%d fallback_migratetype=%d fragmenting=%d change_ownership=%d", pfn_to_page(__entry->pfn), __entry->pfn, __entry->alloc_order,
include/trace/events/pagemap.h renamed from include/trace/events/pagemap.h~mm-tracing-unify-pfn-format-strings +2 -2
@@ -46,7 +46,7 @@ TRACE_EVENT(mm_lru_insertion, ),/* Flag format is based on page-types.c formatting for pagemap */ - TP_printk("page=%p pfn=%lu lru=%d flags=%s%s%s%s%s%s", + TP_printk("page=%p pfn=0x%lx lru=%d flags=%s%s%s%s%s%s", __entry->page, __entry->pfn, __entry->lru,
@@ -75,7 +75,7 @@ TRACE_EVENT(mm_lru_activate, ),/* Flag format is based on page-types.c formatting for pagemap */ -TP_printk ("page=%p pfn=%lu" , __entry->page, __entry->pfn) +TP_printk ("page=%p pfn=0x%lx" , __entry->page, __entry->pfn) );
include/trace/events/page_pool.h renamed from include/trace/events/page_pool.h~mm-tracing-unify-pfn-format-strings +2 -2
@@ -60,7 +60,7 @@ TRACE_EVENT(page_pool_state_release, __entry->pfn =page_to_pfn (page); ), -TP_printk ("page_pool=%p page=%p pfn=%lu release=%u" , +TP_printk ("page_pool=%p page=%p pfn=0x%lx release=%u" , __entry->pool, __entry->page, __entry->pfn, __entry->release) );
@@ -85,7 +85,7 @@ TRACE_EVENT(page_pool_state_hold, __entry->pfn =page_to_pfn (page); ), -TP_printk ("page_pool=%p page=%p pfn=%lu hold=%u" , +TP_printk ("page_pool=%p page=%p pfn=0x%lx hold=%u" , __entry->pool, __entry->page, __entry->pfn, __entry->hold) );
include/trace/events/vmscan.h renamed from include/trace/events/vmscan.h~mm-tracing-unify-pfn-format-strings +1 -1
@@ -330,7 +330,7 @@ TRACE_EVENT(mm_vmscan_writepage, page_is_file_lru(page)); ), -TP_printk ("page=%p pfn=%lu flags=%s" , +TP_printk ("page=%p pfn=0x%lx flags=%s" ,pfn_to_page (__entry->pfn), __entry->pfn,show_reclaim_flags (__entry->reclaim_flags))
_
Andrew Morton
maintainerPATCH 132/192 mm/page_alloc: add an alloc_pages_bulk_array_node() helper
In reply to Andrew MortonFrom: "Uladzislau Rezki (Sony)" <[email protected]>
Subject: mm/page_alloc: add an alloc_pages_bulk_array_node() helper
Patch series "vmalloc() vs bulk allocator", v2.
This patch (of 3):
Add a "node" variant of the alloc_pages_bulk_array() function. The helper guarantees that a __alloc_pages_bulk() is invoked with a valid NUMA node ID.
---
include/linux/gfp.h | 9 +++++++++ 1 file changed, 9 insertions(+)
include/linux/gfp.h renamed from include/linux/gfp.h~mm-page_alloc-add-an-alloc_pages_bulk_array_node-helper +9 -0
@@ -536,6 +536,15 @@ alloc_pages_bulk_array(gfp_t gfp, unsignreturn __alloc_pages_bulk (gfp,numa_mem_id (),NULL , nr_pages,NULL , page_array); } +static inline unsigned long +alloc_pages_bulk_array_node (gfp_t gfp,int nid,unsigned long nr_pages,struct page **page_array) +{ +if (nid ==NUMA_NO_NODE ) + nid =numa_mem_id (); + +return __alloc_pages_bulk (gfp, nid,NULL , nr_pages,NULL , page_array); +} +/* * Allocate pages, preferring the node given as nid. The node must be valid and * online. For more general interface, see alloc_pages_node().
_
Andrew Morton
maintainerPATCH 133/192 mm/vmalloc: switch to bulk allocator in __vmalloc_area_node()
In reply to Andrew MortonFrom: "Uladzislau Rezki (Sony)" <[email protected]>
Subject: mm/vmalloc: switch to bulk allocator in __vmalloc_area_node()
Recently there has been introduced a page bulk allocator for users which need to get number of pages per one call request.
For order-0 pages switch to an alloc_pages_bulk_array_node() instead of alloc_pages_node(), the reason is the former is not capable of allocating set of pages, thus a one call is per one page.
Second, according to my tests the bulk allocator uses less cycles even for scenarios when only one page is requested. Running the "perf" on same test case shows below difference:
<default>
- 45.18% __vmalloc_node
- __vmalloc_node_range
- 35.60% __alloc_pages
- get_page_from_freelist
3.36% __list_del_entry_valid
3.00% check_preemption_disabled
1.42% prep_new_page
<default>
<patch>
- 31.00% __vmalloc_node
- __vmalloc_node_range
- 14.48% __alloc_pages_bulk
3.22% __list_del_entry_valid
- 0.83% __alloc_pages
get_page_from_freelist
<patch>
The "test_vmalloc.sh" also shows performance improvements:
fix_size_alloc_test_4MB loops: 1000000 avg: 89105095 usecfix_size_alloc_test loops: 1000000 avg: 513672 usecfull_fit_alloc_test loops: 1000000 avg: 748900 useclong_busy_list_alloc_test loops: 1000000 avg: 8043038 usecrandom_size_alloc_test loops: 1000000 avg: 4028582 usecfix_align_alloc_test loops: 1000000 avg: 1457671 usec
fix_size_alloc_test_4MB loops: 1000000 avg: 62083711 usecfix_size_alloc_test loops: 1000000 avg: 449207 usecfull_fit_alloc_test loops: 1000000 avg: 735985 useclong_busy_list_alloc_test loops: 1000000 avg: 5176052 usecrandom_size_alloc_test loops: 1000000 avg: 2589252 usecfix_align_alloc_test loops: 1000000 avg: 1365009 usec
For example 4MB allocations illustrates ~30% gain, all the rest is also better.
---
mm/vmalloc.c | 76 +++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 34 deletions(-)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-switch-to-bulk-allocator-in-__vmalloc_area_node +42 -34
@@ -2768,8 +2768,6 @@ static void *__vmalloc_area_node(structunsigned long array_size;unsigned int nr_small_pages = size >>PAGE_SHIFT ;unsigned int page_order; -struct page **pages; -unsigned int i; array_size = (unsigned long )nr_small_pages *sizeof (struct page *); gfp_mask |= __GFP_NOWARN;
@@ -2778,13 +2776,13 @@ static void *__vmalloc_area_node(struct/* Please note that the recursion is strictly bounded. */ if (array_size >PAGE_SIZE ) { - pages =__vmalloc_node (array_size,1 , nested_gfp, node, + area->pages =__vmalloc_node (array_size,1 , nested_gfp, node, area->caller); }else { - pages =kmalloc_node (array_size, nested_gfp, node); + area->pages =kmalloc_node (array_size, nested_gfp, node); } -if (!pages) { +if (!area->pages) {free_vm_area (area);warn_alloc (gfp_mask,NULL , "vmalloc size %lu allocation failure: "
@@ -2793,43 +2791,53 @@ static void *__vmalloc_area_node(structreturn NULL ; } - area->pages = pages; - area->nr_pages = nr_small_pages; + area->nr_pages =0 ;set_vm_area_page_order (area, page_shift -PAGE_SHIFT ); - page_order =vm_area_page_order (area); -/* -* Careful, we allocate and map page_order pages, but tracking is done -* per PAGE_SIZE page so as to keep the vm_struct APIs independent of -* the physical/mapped size. -*/ -for (i =0 ; i < area->nr_pages; i +=1U << page_order) { -struct page *page; -int p; - -/* Compound pages required for remap_vmalloc_page */ - page =alloc_pages_node (node, gfp_mask | __GFP_COMP, page_order); -if (unlikely (!page)) { -/* Successfully allocated i pages, free them in __vfree() */ - area->nr_pages = i; -atomic_long_add (area->nr_pages, &nr_vmalloc_pages); -warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"page order %u allocation failed" , - area->nr_pages *PAGE_SIZE , page_order); - goto fail; - } +if (!page_order) { + area->nr_pages =alloc_pages_bulk_array_node ( + gfp_mask, node, nr_small_pages, area->pages); + }else { +/* +* Careful, we allocate and map page_order pages, but tracking is done +* per PAGE_SIZE page so as to keep the vm_struct APIs independent of +* the physical/mapped size. +*/ +while (area->nr_pages < nr_small_pages) { +struct page *page; +int i; + +/* Compound pages required for remap_vmalloc_page */ + page =alloc_pages_node (node, gfp_mask | __GFP_COMP, page_order); +if (unlikely (!page)) +break ; + +for (i =0 ; i < (1U << page_order); i++) + area->pages[area->nr_pages + i] = page + i; -for (p =0 ; p < (1U << page_order); p++) - area->pages[i + p] = page + p; +if (gfpflags_allow_blocking (gfp_mask)) +cond_resched (); -if (gfpflags_allow_blocking (gfp_mask)) -cond_resched (); + area->nr_pages +=1U << page_order; + } } +atomic_long_add (area->nr_pages, &nr_vmalloc_pages); -if (vmap_pages_range (addr, addr + size, prot, pages, page_shift) <0 ) { +/* +* If not enough pages were obtained to accomplish an +* allocation request, free them via __vfree() if any. +*/ +if (area->nr_pages != nr_small_pages) { +warn_alloc (gfp_mask,NULL , +"vmalloc size %lu allocation failure: " +"page order %u allocation failed" , + area->nr_pages *PAGE_SIZE , page_order); + goto fail; + } + +if (vmap_pages_range (addr, addr + size, prot, area->pages, page_shift) <0 ) { warn_alloc(gfp_mask, NULL, "vmalloc size %lu allocation failure: " "failed to map pages",
_
Andrew Morton
maintainerPATCH 134/192 mm/vmalloc: print a warning message first on failure
In reply to Andrew MortonFrom: "Uladzislau Rezki (Sony)" <[email protected]>
Subject: mm/vmalloc: print a warning message first on failure
When a memory allocation for array of pages are not succeed emit a warning message as a first step and then perform the further cleanup.
The reason it should be done in a right order is the clean up function which is free_vm_area() can potentially also follow its error paths what can lead to confusion what was broken first.
---
mm/vmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-print-a-warning-message-first-on-failure +1 -1
@@ -2783,11 +2783,11 @@ static void *__vmalloc_area_node(struct }if (!area->pages) { -free_vm_area (area);warn_alloc (gfp_mask,NULL ,"vmalloc size %lu allocation failure: " "page array size %lu allocation failed" , nr_small_pages *PAGE_SIZE , array_size); +free_vm_area (area);return NULL ; }
_
Andrew Morton
maintainerPATCH 135/192 mm/vmalloc: remove quoted strings split across lines
In reply to Andrew MortonFrom: "Uladzislau Rezki (Sony)" <[email protected]>
Subject: mm/vmalloc: remove quoted strings split across lines
A checkpatch.pl script complains on splitting a text across lines. It is because if a user wants to find an entire string he or she will not succeeded.
<snip>
WARNING: quoted string split across lines
+ "vmalloc size %lu allocation failure: "
+ "page order %u allocation failed",
total: 0 errors, 1 warnings, 10 lines checked
<snip>
---
mm/vmalloc.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-remove-quoted-string-split-across-lines +9 -12
@@ -2784,9 +2784,8 @@ static void *__vmalloc_area_node(structif (!area->pages) {warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"page array size %lu allocation failed" , - nr_small_pages *PAGE_SIZE , array_size); +"vmalloc error: size %lu, failed to allocated page array size %lu" , + nr_small_pages *PAGE_SIZE , array_size);free_vm_area (area);return NULL ; }
@@ -2831,17 +2830,15 @@ static void *__vmalloc_area_node(struct*/ if (area->nr_pages != nr_small_pages) {warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"page order %u allocation failed" , +"vmalloc error: size %lu, page order %u, failed to allocate pages" , area->nr_pages *PAGE_SIZE , page_order); goto fail; }if (vmap_pages_range (addr, addr + size, prot, area->pages, page_shift) <0 ) {warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"failed to map pages" , - area->nr_pages *PAGE_SIZE ); +"vmalloc error: size %lu, failed to map pages" , + area->nr_pages *PAGE_SIZE ); goto fail; }
@@ -2886,8 +2883,8 @@ void *__vmalloc_node_range(unsigned longif ((size >>PAGE_SHIFT ) >totalram_pages ()) {warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"exceeds total pages" , real_size); +"vmalloc error: size %lu, exceeds total pages" , + real_size);return NULL ; }
@@ -2918,8 +2915,8 @@ again: gfp_mask, caller);if (!area) {warn_alloc (gfp_mask,NULL , -"vmalloc size %lu allocation failure: " -"vm_struct allocation failed" , real_size); +"vmalloc error: size %lu, vm_struct allocation failed" , + real_size); goto fail; }
_
Andrew Morton
maintainerPATCH 136/192 mm/vmalloc: fallback to a single page allocator
In reply to Andrew MortonFrom: Uladzislau Rezki <[email protected]>
Subject: mm/vmalloc: fallback to a single page allocator
Currently for order-0 pages we use a bulk-page allocator to get set of pages. From the other hand not allocating all pages is something that might occur. In that case we should fallbak to the single-page allocator trying to get missing pages, because it is more permissive(direct reclaim, etc).
Introduce a vm_area_alloc_pages() function where the described logic is implemented.
---
mm/vmalloc.c | 81 +++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 29 deletions(-)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-fallback-to-a-single-page-allocator +52 -29
@@ -2758,6 +2758,54 @@ void *vmap_pfn(unsigned long *pfns, unsiEXPORT_SYMBOL_GPL (vmap_pfn);#endif /* CONFIG_VMAP_PFN */ +static inline unsigned int +vm_area_alloc_pages (gfp_t gfp,int nid, +unsigned int order,unsigned long nr_pages,struct page **pages) +{ +unsigned int nr_allocated =0 ; + +/* +* For order-0 pages we make use of bulk allocator, if +* the page array is partly or not at all populated due +* to fails, fallback to a single page allocator that is +* more permissive. +*/ +if (!order) + nr_allocated =alloc_pages_bulk_array_node ( + gfp, nid, nr_pages, pages); +else +/* +* Compound pages required for remap_vmalloc_page if +* high-order pages. +*/ + gfp |= __GFP_COMP; + +/* High-order pages or fallback path if "bulk" fails. */ +while (nr_allocated < nr_pages) { +struct page *page; +int i; + + page =alloc_pages_node (nid, gfp, order); +if (unlikely (!page)) +break ; + +/* +* Careful, we allocate and map page-order pages, but +* tracking is done per PAGE_SIZE page so as to keep the +* vm_struct APIs independent of the physical/mapped size. +*/ +for (i =0 ; i < (1U << order); i++) + pages[nr_allocated + i] = page + i; + +if (gfpflags_allow_blocking (gfp)) +cond_resched (); + + nr_allocated +=1U << order; + } + +return nr_allocated; +} +static void *__vmalloc_area_node (struct vm_struct *area,gfp_t gfp_mask,pgprot_t prot,unsigned int page_shift,int node)
@@ -2790,37 +2838,11 @@ static void *__vmalloc_area_node(structreturn NULL ; } - area->nr_pages =0 ;set_vm_area_page_order (area, page_shift -PAGE_SHIFT ); page_order =vm_area_page_order (area); -if (!page_order) { - area->nr_pages =alloc_pages_bulk_array_node ( - gfp_mask, node, nr_small_pages, area->pages); - }else { -/* -* Careful, we allocate and map page_order pages, but tracking is done -* per PAGE_SIZE page so as to keep the vm_struct APIs independent of -* the physical/mapped size. -*/ -while (area->nr_pages < nr_small_pages) { -struct page *page; -int i; - -/* Compound pages required for remap_vmalloc_page */ - page =alloc_pages_node (node, gfp_mask | __GFP_COMP, page_order); -if (unlikely (!page)) -break ; - -for (i =0 ; i < (1U << page_order); i++) - area->pages[area->nr_pages + i] = page + i; - -if (gfpflags_allow_blocking (gfp_mask)) -cond_resched (); - - area->nr_pages +=1U << page_order; - } - } + area->nr_pages =vm_area_alloc_pages (gfp_mask, node, + page_order, nr_small_pages, area->pages);atomic_long_add (area->nr_pages, &nr_vmalloc_pages);
@@ -2835,7 +2857,8 @@ static void *__vmalloc_area_node(struct goto fail; } -if (vmap_pages_range (addr, addr + size, prot, area->pages, page_shift) <0 ) { +if (vmap_pages_range (addr, addr + size, prot, area->pages, + page_shift) <0 ) {warn_alloc (gfp_mask,NULL ,"vmalloc error: size %lu, failed to map pages" , area->nr_pages *PAGE_SIZE );
_
Andrew Morton
maintainerPATCH 137/192 mm: vmalloc: add cond_resched() in __vunmap()
In reply to Andrew MortonFrom: Rafael Aquini <[email protected]>
Subject: mm: vmalloc: add cond_resched() in __vunmap()
On non-preemptible kernel builds the watchdog can complain about soft lockups when vfree() is called against large vmalloc areas:
[ 210.851798] kvmalloc-test: vmalloc(2199023255552) succeeded [ 238.654842] watchdog: BUG: soft lockup - CPU#181 stuck for 26s! [rmmod:5203] [ 238.662716] Modules linked in: kvmalloc_test(OE-) ... [ 238.772671] CPU: 181 PID: 5203 Comm: rmmod Tainted: G S OE 5.13.0-rc7+ #1 [ 238.781413] Hardware name: Intel Corporation PURLEY/PURLEY, BIOS PLYXCRB1.86B.0553.D01.1809190614 09/19/2018 [ 238.792383] RIP: 0010:free_unref_page+0x52/0x60 [ 238.797447] Code: 48 c1 fd 06 48 89 ee e8 9c d0 ff ff 84 c0 74 19 9c 41 5c fa 48 89 ee 48 89 df e8 b9 ea ff ff 41 f7 c4 00 02 00 00 74 01 fb 5b <5d> 41 5c c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 f0 29 77 [ 238.818406] RSP: 0018:ffffb4d87868fe98 EFLAGS: 00000206 [ 238.824236] RAX: 0000000000000000 RBX: 000000001da0c945 RCX: ffffb4d87868fe40 [ 238.832200] RDX: ffffd79d3beed108 RSI: ffffd7998501dc08 RDI: ffff9c6fbffd7010 [ 238.840166] RBP: 000000000d518cbd R08: ffffd7998501dc08 R09: 0000000000000001 [ 238.848131] R10: 0000000000000000 R11: ffffd79d3beee088 R12: 0000000000000202 [ 238.856095] R13: ffff9e5be3eceec0 R14: 0000000000000000 R15: 0000000000000000 [ 238.864059] FS: 00007fe082c2d740(0000) GS:ffff9f4c69b40000(0000) knlGS:0000000000000000 [ 238.873089] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 238.879503] CR2: 000055a000611128 CR3: 000000f6094f6006 CR4: 00000000007706e0 [ 238.887467] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 238.895433] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 238.903397] PKRU: 55555554 [ 238.906417] Call Trace: [ 238.909149] __vunmap+0x17c/0x220 [ 238.912851] __x64_sys_delete_module+0x13a/0x250 [ 238.918008] ? syscall_trace_enter.isra.20+0x13c/0x1b0 [ 238.923746] do_syscall_64+0x39/0x80 [ 238.927740] entry_SYSCALL_64_after_hwframe+0x44/0xae
Like in other range zapping routines that iterate over a large list, lets just add cond_resched() within __vunmap()'s page-releasing loop in order to avoid the watchdog splats.
---
mm/vmalloc.c | 1 + 1 file changed, 1 insertion(+)
mm/vmalloc.c renamed from mm/vmalloc.c~mm-vmalloc-add-cond_resched-in-__vunmap +1 -0
@@ -2567,6 +2567,7 @@ static void __vunmap(const void *addr, iBUG_ON (!page);__free_pages (page, page_order); +cond_resched (); }atomic_long_sub (area->nr_pages, &nr_vmalloc_pages);
_
Andrew Morton
maintainerPATCH 138/192 printk: introduce dump_stack_lvl()
In reply to Andrew MortonFrom: Alexander Potapenko <[email protected]>
Subject: printk: introduce dump_stack_lvl()
dump_stack() is used for many different cases, which may require a log level consistent with other kernel messages surrounding the dump_stack() call. Without that, certain systems that are configured to ignore the default level messages will miss stack traces in critical error reports.
This patch introduces dump_stack_lvl() that behaves similarly to dump_stack(), but accepts a custom log level. The old dump_stack() becomes equal to dump_stack_lvl(KERN_DEFAULT).
A somewhat similar patch has been proposed in 2012:
https://lore.kernel.org/lkml/1332493269.2359.9.camel@hebo/ , but wasn't merged.
---
include/linux/printk.h | 5 +++++ lib/dump_stack.c | 20 +++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-)
include/linux/printk.h renamed from include/linux/printk.h~printk-introduce-dump_stack_lvl +5 -0
@@ -206,6 +206,7 @@ void __init setup_log_buf(int early); __printf(1, 2) void dump_stack_set_arch_desc(const char *fmt, ...);void dump_stack_print_info (const char *log_lvl);void show_regs_print_info (const char *log_lvl); +extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold; extern asmlinkage void dump_stack(void) __cold;extern void printk_safe_flush (void );extern void printk_safe_flush_on_panic (void );
@@ -269,6 +270,10 @@ static inline void show_regs_print_info( { } +static inline void dump_stack_lvl (const char *log_lvl) +{ +} +static inline void dump_stack (void ) { }
lib/dump_stack.c renamed from lib/dump_stack.c~printk-introduce-dump_stack_lvl +13 -7
@@ -73,10 +73,10 @@ void show_regs_print_info(const char *lodump_stack_print_info (log_lvl); } -static void __dump_stack (void ) +static void __dump_stack (const char *log_lvl) { -dump_stack_print_info (KERN_DEFAULT ); -show_stack (NULL ,NULL ,KERN_DEFAULT ); +dump_stack_print_info (log_lvl); +show_stack (NULL ,NULL , log_lvl); }/**
@@ -87,7 +87,7 @@ static void __dump_stack(void)#ifdef CONFIG_SMP static atomic_t dump_lock =ATOMIC_INIT (-1 ); -asmlinkage __visible void dump_stack(void) +asmlinkage __visible void dump_stack_lvl(const char *log_lvl) {unsigned long flags;int was_locked;
@@ -117,7 +117,7 @@ retry: goto retry; } -__dump_stack (); +__dump_stack (log_lvl);if (!was_locked)atomic_set (&dump_lock,-1 );
@@ -125,9 +125,15 @@ retry:local_irq_restore (flags); }#else -asmlinkage __visible void dump_stack(void) +asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { -__dump_stack (); +__dump_stack (log_lvl); }#endif +EXPORT_SYMBOL (dump_stack_lvl); + +asmlinkage __visible void dump_stack(void) +{ +dump_stack_lvl (KERN_DEFAULT ); +}EXPORT_SYMBOL (dump_stack);
_
Andrew Morton
maintainerPATCH 139/192 kasan: use dump_stack_lvl(KERN_ERR) to print stacks
In reply to Andrew MortonFrom: Alexander Potapenko <[email protected]>
Subject: kasan: use dump_stack_lvl(KERN_ERR) to print stacks
Most of the contents of KASAN reports are printed with pr_err(), so use a consistent logging level to print the memory access stacks.
---
mm/kasan/report.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
mm/kasan/report.c renamed from mm/kasan/report.c~kasan-use-dump_stack_lvlkern_err-to-print-stacks +3 -3
@@ -230,7 +230,7 @@ static void print_address_description(vo {struct page *page =kasan_addr_to_page (addr); -dump_stack (); +dump_stack_lvl (KERN_ERR );pr_err ("\n" );if (page &&PageSlab (page)) {
@@ -375,7 +375,7 @@ void kasan_report_async(void)pr_err ("BUG: KASAN: invalid-access\n" );pr_err ("Asynchronous mode enabled: no access details available\n" );pr_err ("\n" ); -dump_stack (); +dump_stack_lvl (KERN_ERR ); end_report(&flags, 0); }#endif /* CONFIG_KASAN_HW_TAGS */
@@ -420,7 +420,7 @@ static void __kasan_report(unsigned longpr_err ("\n" );print_memory_metadata (info.first_bad_addr); }else { -dump_stack (); +dump_stack_lvl (KERN_ERR ); }end_report (&flags, addr);
_
Andrew Morton
maintainerPATCH 140/192 kasan: test: improve failure message in KUNIT_EXPECT_KASAN_FAIL()
In reply to Andrew MortonFrom: David Gow <[email protected]>
Subject: kasan: test: improve failure message in KUNIT_EXPECT_KASAN_FAIL()
The KUNIT_EXPECT_KASAN_FAIL() macro currently uses KUNIT_EXPECT_EQ() to compare fail_data.report_expected and fail_data.report_found. This always gave a somewhat useless error message on failure, but the addition of extra compile-time checking with READ_ONCE() has caused it to get much longer, and be truncated before anything useful is displayed.
Instead, just check fail_data.report_found by hand (we've just set report_expected to 'true'), and print a better failure message with KUNIT_FAIL(). Because of this, report_expected is no longer used anywhere, and can be removed.
Beforehand, a failure in:KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)area)[3100]); would have looked like:
[22:00:34] [FAILED] vmalloc_oob
[22:00:34] # vmalloc_oob: EXPECTATION FAILED at lib/test_kasan.c:991
[22:00:34] Expected ({ do { extern void __compiletime_assert_705(void) __attribute__((__error__("Unsupported access size for {READ,WRITE}_ONCE()."))); if (!((sizeof(fail_data.report_expected) == sizeof(char) || sizeof(fail_data.repp
[22:00:34] not ok 45 - vmalloc_oobWith this change, it instead looks like:
[22:04:04] [FAILED] vmalloc_oob
[22:04:04] # vmalloc_oob: EXPECTATION FAILED at lib/test_kasan.c:993 [22:04:04] KASAN failure expected in "((volatile char *)area)[3100]", but none occurred [22:04:04] not ok 45 - vmalloc_oob
Also update the example failure in the documentation to reflect this.
---
Documentation/dev-tools/kasan.rst | 9 ++++----- include/linux/kasan.h | 1 - lib/test_kasan.c | 11 +++++------ 3 files changed, 9 insertions(+), 12 deletions(-)
Documentation/dev-tools/kasan.rst renamed from Documentation/dev-tools/kasan.rst~kasan-test-improve-failure-message-in-kunit_expect_kasan_fail +4 -5
@@ -447,11 +447,10 @@ When a test fails due to a failed ``kmal When a test fails due to a missing KASAN report:: - # kmalloc_double_kzfree: EXPECTATION FAILED at lib/test_kasan.c:629 - Expected kasan_data->report_expected == kasan_data->report_found, but - kasan_data->report_expected == 1 - kasan_data->report_found == 0 - not ok 28 - kmalloc_double_kzfree + # kmalloc_double_kzfree: EXPECTATION FAILED at lib/test_kasan.c:974 + KASAN failure expected in "kfree_sensitive(ptr)", but none occurred + not ok 44 - kmalloc_double_kzfree + At the end the cumulative status of all KASAN tests is printed. On success::
include/linux/kasan.h renamed from include/linux/kasan.h~kasan-test-improve-failure-message-in-kunit_expect_kasan_fail +0 -1
@@ -17,7 +17,6 @@ struct task_struct;/* kasan_data struct is used in KUnit tests for KASAN expected failures */ struct kunit_kasan_expectation { -bool report_expected;bool report_found; };
lib/test_kasan.c renamed from lib/test_kasan.c~kasan-test-improve-failure-message-in-kunit_expect_kasan_fail +5 -6
@@ -55,7 +55,6 @@ static int kasan_test_init(struct kunit multishot =kasan_save_enable_multi_shot ();kasan_set_tagging_report_once (false); fail_data.report_found = false; - fail_data.report_expected = false;kunit_add_named_resource (test,NULL ,NULL , &resource,"kasan_data" , &fail_data);return 0 ;
@@ -94,20 +93,20 @@ static void kasan_test_exit(struct kunit !kasan_async_mode_enabled()) \migrate_disable (); \KUNIT_EXPECT_FALSE (test,READ_ONCE (fail_data.report_found)); \ -WRITE_ONCE (fail_data.report_expected, true); \barrier (); \ expression; \barrier (); \ -KUNIT_EXPECT_EQ (test, \ -READ_ONCE (fail_data.report_expected), \ -READ_ONCE (fail_data.report_found)); \ +if (!READ_ONCE (fail_data.report_found)) { \ +KUNIT_FAIL (test,KUNIT_SUBTEST_INDENT "KASAN failure " \ + "expected in \"" #expression \ +"\", but none occurred" ); \ + } \if (IS_ENABLED (CONFIG_KASAN_HW_TAGS )) { \if (READ_ONCE (fail_data.report_found)) \kasan_enable_tagging_sync (); \migrate_enable (); \ } \WRITE_ONCE (fail_data.report_found, false); \ -WRITE_ONCE (fail_data.report_expected, false); \ } while (0)#define KASAN_TEST_NEEDS_CONFIG_ON (test, config) do { \
_
Andrew Morton
maintainerPATCH 141/192 kasan: allow an architecture to disable inline instrumentation
In reply to Andrew MortonFrom: Daniel Axtens <[email protected]>
Subject: kasan: allow an architecture to disable inline instrumentation
Patch series "KASAN core changes for ppc64 radix KASAN", v16.
Building on the work of Christophe, Aneesh and Balbir, I've ported KASAN to 64-bit Book3S kernels running on the Radix MMU. I've been trying this for a while, but we keep having collisions between the kasan code in the mm tree and the code I want to put in to the ppc tree.
This series just contains the kasan core changes that we need. There should be no noticeable changes to other platforms.
This patch (of 4):
For annoying architectural reasons, it's very difficult to support inline instrumentation on powerpc64.*
Add a Kconfig flag to allow an arch to disable inline. (It's a bit annoying to be 'backwards', but I'm not aware of any way to have an arch force a symbol to be 'n', rather than 'y'.)
We also disable stack instrumentation in this case as it does things that are functionally equivalent to inline instrumentation, namely adding code that touches the shadow directly without going through a C helper.
* on ppc64 atm, the shadow lives in virtual memory and isn't accessible in real mode. However, before we turn on virtual memory, we parse the device tree to determine which platform and MMU we're running under. That calls generic DT code, which is instrumented. Inline instrumentation in DT would unconditionally attempt to touch the shadow region, which we won't have set up yet, and would crash. We can make outline mode wait for the arch to be ready, but we can't change what the compiler inserts for inline mode.
---
lib/Kconfig.kasan | 12 ++++++++++++ 1 file changed, 12 insertions(+)
lib/Kconfig.kasan renamed from lib/Kconfig.kasan~kasan-allow-an-architecture-to-disable-inline-instrumentation +12 -0
@@ -12,6 +12,13 @@ config HAVE_ARCH_KASAN_HW_TAGS config HAVE_ARCH_KASAN_VMALLOC bool +config ARCH_DISABLE_KASAN_INLINE + bool + help + An architecture might not support inline instrumentation. + When this option is selected, inline and stack instrumentation are + disabled. + config CC_HAS_KASAN_GENERIC def_bool $(cc-option, -fsanitize=kernel-address)
@@ -130,6 +137,7 @@ config KASAN_OUTLINE config KASAN_INLINE bool "Inline instrumentation" + depends on !ARCH_DISABLE_KASAN_INLINE help Compiler directly inserts code checking shadow memory before memory accesses. This is faster than outline (in some workloads
@@ -141,6 +149,7 @@ endchoice config KASAN_STACK bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST depends on KASAN_GENERIC || KASAN_SW_TAGS + depends on !ARCH_DISABLE_KASAN_INLINE default y if CC_IS_GCC help The LLVM stack address sanitizer has a know problem that
@@ -154,6 +163,9 @@ config KASAN_STACK but clang users can still enable it for builds without CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe to use and enabled by default. + If the architecture disables inline instrumentation, stack + instrumentation is also disabled as it adds inline-style + instrumentation that is run unconditionally. config KASAN_SW_TAGS_IDENTIFY bool "Enable memory corruption identification"
_
Andrew Morton
maintainerPATCH 142/192 kasan: allow architectures to provide an outline readiness check
In reply to Andrew MortonFrom: Daniel Axtens <[email protected]>
Subject: kasan: allow architectures to provide an outline readiness check
Allow architectures to define a kasan_arch_is_ready() hook that bails out of any function that's about to touch the shadow unless the arch says that it is ready for the memory to be accessed. This is fairly uninvasive and should have a negligible performance penalty.
This will only work in outline mode, so an arch must specify ARCH_DISABLE_KASAN_INLINE if it requires this.
---
mm/kasan/common.c | 3 +++ mm/kasan/generic.c | 3 +++ mm/kasan/kasan.h | 6 ++++++ mm/kasan/shadow.c | 6 ++++++ 4 files changed, 18 insertions(+)
mm/kasan/common.c renamed from mm/kasan/common.c~kasan-allow-architectures-to-provide-an-outline-readiness-check +3 -0
@@ -331,6 +331,9 @@ static inline bool ____kasan_slab_free(su8 tag;void *tagged_object; +if (!kasan_arch_is_ready ()) +return false; + tag =get_tag (object); tagged_object = object; object =kasan_reset_tag (object);
mm/kasan/generic.c renamed from mm/kasan/generic.c~kasan-allow-architectures-to-provide-an-outline-readiness-check +3 -0
@@ -163,6 +163,9 @@ static __always_inline bool check_region size_t size, bool write, unsigned long ret_ip) { + if (!kasan_arch_is_ready()) + return true; +if (unlikely (size ==0 ))return true;
mm/kasan/kasan.h renamed from mm/kasan/kasan.h~kasan-allow-architectures-to-provide-an-outline-readiness-check +6 -0
@@ -449,6 +449,12 @@ static inline void kasan_poison_last_gra#endif /* CONFIG_KASAN_GENERIC */ +#ifndef kasan_arch_is_ready +static inline bool kasan_arch_is_ready (void ) {return true; } +#elif !defined(CONFIG_KASAN_GENERIC ) || !defined(CONFIG_KASAN_OUTLINE ) +#error kasan_arch_is_ready only works in KASAN generic outline mode! +#endif +/* * Exported functions for interfaces called from assembly or from generated * code. Declarations here to avoid warning about missing declarations.
mm/kasan/shadow.c renamed from mm/kasan/shadow.c~kasan-allow-architectures-to-provide-an-outline-readiness-check +6 -0
@@ -73,6 +73,9 @@ void kasan_poison(const void *addr, size {void *shadow_start, *shadow_end; +if (!kasan_arch_is_ready ()) +return ; +/* * Perform shadow offset calculation based on untagged address, as * some of the callers (e.g. kasan_poison_object_data) pass tagged
@@ -99,6 +102,9 @@ EXPORT_SYMBOL(kasan_poison);#ifdef CONFIG_KASAN_GENERIC void kasan_poison_last_granule (const void *addr,size_t size) { +if (!kasan_arch_is_ready ()) +return ; +if (size &KASAN_GRANULE_MASK ) {u8 *shadow = (u8 *)kasan_mem_to_shadow (addr + size); *shadow = size &KASAN_GRANULE_MASK ;
_
Andrew Morton
maintainerPATCH 143/192 mm: define default MAX_PTRS_PER_* in include/pgtable.h
In reply to Andrew MortonFrom: Daniel Axtens <[email protected]>
Subject: mm: define default MAX_PTRS_PER_* in include/pgtable.h
Commit c65e774fb3f6 ("x86/mm: Make PGDIR_SHIFT and PTRS_PER_P4D variable") made PTRS_PER_P4D variable on x86 and introduced MAX_PTRS_PER_P4D as a constant for cases which need a compile-time constant (e.g. fixed-size arrays).
powerpc likewise has boot-time selectable MMU features which can cause other mm "constants" to vary. For KASAN, we have some static PTE/PMD/PUD/P4D arrays so we need compile-time maximums for all these constants. Extend the MAX_PTRS_PER_ idiom, and place default definitions in include/pgtable.h. These define MAX_PTRS_PER_x to be PTRS_PER_x unless an architecture has defined MAX_PTRS_PER_x in its arch headers.
Clean up pgtable-nop4d.h and s390's MAX_PTRS_PER_P4D definitions while we're at it: both can just pick up the default now.
---
arch/s390/include/asm/pgtable.h | 2 -- include/asm-generic/pgtable-nop4d.h | 1 - include/linux/pgtable.h | 22 ++++++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-)
arch/s390/include/asm/pgtable.h renamed from arch/s390/include/asm/pgtable.h~mm-define-default-max_ptrs_per_-in-include-pgtableh +0 -2
@@ -344,8 +344,6 @@ static inline int is_module_addr(void *a#define PTRS_PER_P4D _CRST_ENTRIES#define PTRS_PER_PGD _CRST_ENTRIES -#define MAX_PTRS_PER_P4D PTRS_PER_P4D -/* * Segment table and region3 table entry encoding * (R = read-only, I = invalid, y = young bit):
include/asm-generic/pgtable-nop4d.h renamed from include/asm-generic/pgtable-nop4d.h~mm-define-default-max_ptrs_per_-in-include-pgtableh +0 -1
@@ -9,7 +9,6 @@typedef struct {pgd_t pgd; }p4d_t ;#define P4D_SHIFT PGDIR_SHIFT -#define MAX_PTRS_PER_P4D 1#define PTRS_PER_P4D 1#define P4D_SIZE (1UL << P4D_SHIFT)#define P4D_MASK (~(P4D_SIZE-1))
include/linux/pgtable.h renamed from include/linux/pgtable.h~mm-define-default-max_ptrs_per_-in-include-pgtableh +22 -0
@@ -1592,4 +1592,26 @@ typedef unsigned int pgtbl_mod_mask;#define pte_leaf_size (x) PAGE_SIZE#endif +/* +* Some architectures have MMUs that are configurable or selectable at boot +* time. These lead to variable PTRS_PER_x. For statically allocated arrays it +* helps to have a static maximum value. +*/ + +#ifndef MAX_PTRS_PER_PTE +#define MAX_PTRS_PER_PTE PTRS_PER_PTE +#endif + +#ifndef MAX_PTRS_PER_PMD +#define MAX_PTRS_PER_PMD PTRS_PER_PMD +#endif + +#ifndef MAX_PTRS_PER_PUD +#define MAX_PTRS_PER_PUD PTRS_PER_PUD +#endif + +#ifndef MAX_PTRS_PER_P4D +#define MAX_PTRS_PER_P4D PTRS_PER_P4D +#endif +#endif /* _LINUX_PGTABLE_H */
_
Andrew Morton
maintainerPATCH 144/192 kasan: use MAX_PTRS_PER_* for early shadow tables
In reply to Andrew MortonFrom: Daniel Axtens <[email protected]>
Subject: kasan: use MAX_PTRS_PER_* for early shadow tables
powerpc has a variable number of PTRS_PER_*, set at runtime based on the MMU that the kernel is booted under.
This means the PTRS_PER_* are no longer constants, and therefore breaks the build. Switch to using MAX_PTRS_PER_*, which are constant.
---
include/linux/kasan.h | 6 +++--- mm/kasan/init.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)
include/linux/kasan.h renamed from include/linux/kasan.h~kasan-use-max_ptrs_per_-for-early-shadow-tables +3 -3
@@ -40,9 +40,9 @@ struct kunit_kasan_expectation {#endif extern unsigned char kasan_early_shadow_page[PAGE_SIZE ]; -extern pte_t kasan_early_shadow_pte[PTRS_PER_PTE +PTE_HWTABLE_PTRS ]; -extern pmd_t kasan_early_shadow_pmd[PTRS_PER_PMD ]; -extern pud_t kasan_early_shadow_pud[PTRS_PER_PUD ]; +extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE +PTE_HWTABLE_PTRS ]; +extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD ]; +extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD ];extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D ]; int kasan_populate_early_shadow(const void *shadow_start,
mm/kasan/init.c renamed from mm/kasan/init.c~kasan-use-max_ptrs_per_-for-early-shadow-tables +3 -3
@@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t }#endif #if CONFIG_PGTABLE_LEVELS >3 -pud_t kasan_early_shadow_pud[PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss;static inline bool kasan_pud_table (p4d_t p4d) {return p4d_page (p4d) ==virt_to_page (lm_alias (kasan_early_shadow_pud));
@@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t }#endif #if CONFIG_PGTABLE_LEVELS >2 -pmd_t kasan_early_shadow_pmd[PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss;static inline bool kasan_pmd_table (pud_t pud) {return pud_page (pud) ==virt_to_page (lm_alias (kasan_early_shadow_pmd));
@@ -64,7 +64,7 @@ static inline bool kasan_pmd_table(pud_treturn false; }#endif -pte_t kasan_early_shadow_pte[PTRS_PER_PTE + PTE_HWTABLE_PTRS] +pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] __page_aligned_bss;static inline bool kasan_pte_table (pmd_t pmd)
_
Andrew Morton
maintainerPATCH 145/192 kasan: rename CONFIG_KASAN_SW_TAGS_IDENTIFY to CONFIG_KASAN_TAGS_IDENTIFY
In reply to Andrew MortonFrom: Kuan-Ying Lee <[email protected]>
Subject: kasan: rename CONFIG_KASAN_SW_TAGS_IDENTIFY to CONFIG_KASAN_TAGS_IDENTIFY
Patch series "kasan: add memory corruption identification support for hw tag-based kasan", v4.
Add memory corruption identification for hardware tag-based KASAN mode.
This patch (of 3):
Rename CONFIG_KASAN_SW_TAGS_IDENTIFY to CONFIG_KASAN_TAGS_IDENTIFY in order to be compatible with hardware tag-based mode.
---
lib/Kconfig.kasan | 2 +- mm/kasan/kasan.h | 4 ++-- mm/kasan/report_sw_tags.c | 2 +- mm/kasan/sw_tags.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-)
lib/Kconfig.kasan renamed from lib/Kconfig.kasan~kasan-rename-config_kasan_sw_tags_identify-to-config_kasan_tags_identify +1 -1
@@ -167,7 +167,7 @@ config KASAN_STACK instrumentation is also disabled as it adds inline-style instrumentation that is run unconditionally. -config KASAN_SW_TAGS_IDENTIFY +config KASAN_TAGS_IDENTIFY bool "Enable memory corruption identification" depends on KASAN_SW_TAGS help
mm/kasan/kasan.h renamed from mm/kasan/kasan.h~kasan-rename-config_kasan_sw_tags_identify-to-config_kasan_tags_identify +2 -2
@@ -153,7 +153,7 @@ struct kasan_track {depot_stack_handle_t stack; }; -#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY +#ifdef CONFIG_KASAN_TAGS_IDENTIFY #define KASAN_NR_FREE_STACKS 5#else #define KASAN_NR_FREE_STACKS 1
@@ -170,7 +170,7 @@ struct kasan_alloc_meta {#else struct kasan_track free_track[KASAN_NR_FREE_STACKS ];#endif -#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY +#ifdef CONFIG_KASAN_TAGS_IDENTIFY u8 free_pointer_tag[KASAN_NR_FREE_STACKS ];u8 free_track_idx;#endif
mm/kasan/report_sw_tags.c renamed from mm/kasan/report_sw_tags.c~kasan-rename-config_kasan_sw_tags_identify-to-config_kasan_tags_identify +1 -1
@@ -31,7 +31,7 @@const char *kasan_get_bug_type (struct kasan_access_info *info) { -#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY +#ifdef CONFIG_KASAN_TAGS_IDENTIFY struct kasan_alloc_meta *alloc_meta;struct kmem_cache *cache;struct page *page;
mm/kasan/sw_tags.c renamed from mm/kasan/sw_tags.c~kasan-rename-config_kasan_sw_tags_identify-to-config_kasan_tags_identify +2 -2
@@ -177,7 +177,7 @@ void kasan_set_free_info(struct kmem_cacif (!alloc_meta)return ; -#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY +#ifdef CONFIG_KASAN_TAGS_IDENTIFY idx = alloc_meta->free_track_idx; alloc_meta->free_pointer_tag[idx] = tag; alloc_meta->free_track_idx = (idx +1 ) %KASAN_NR_FREE_STACKS ;
@@ -196,7 +196,7 @@ struct kasan_track *kasan_get_free_trackif (!alloc_meta)return NULL ; -#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY +#ifdef CONFIG_KASAN_TAGS_IDENTIFY for (i =0 ; i <KASAN_NR_FREE_STACKS ; i++) {if (alloc_meta->free_pointer_tag[i] == tag)break ;
_
Andrew Morton
maintainerPATCH 146/192 kasan: integrate the common part of two KASAN tag-based modes
In reply to Andrew MortonFrom: Kuan-Ying Lee <[email protected]>
Subject: kasan: integrate the common part of two KASAN tag-based modes
1. Move kasan_get_free_track() and kasan_set_free_info() into tags.c and combine these two functions for SW_TAGS and HW_TAGS kasan mode.
2. Move kasan_get_bug_type() to report_tags.c and make this function compatible for SW_TAGS and HW_TAGS kasan mode.
---
mm/kasan/Makefile | 4 +- mm/kasan/hw_tags.c | 22 ------------- mm/kasan/report_hw_tags.c | 5 --- mm/kasan/report_sw_tags.c | 43 -------------------------- mm/kasan/report_tags.c | 51 +++++++++++++++++++++++++++++++ mm/kasan/sw_tags.c | 41 ------------------------- mm/kasan/tags.c | 59 ++++++++++++++++++++++++++++++++++++ 7 files changed, 112 insertions(+), 113 deletions(-)
mm/kasan/hw_tags.c renamed from mm/kasan/hw_tags.c~kasan-integrate-the-common-part-of-two-kasan-tag-based-modes +0 -22
@@ -216,28 +216,6 @@ void __init kasan_init_hw_tags(void)pr_info ("KernelAddressSanitizer initialized\n" ); } -void kasan_set_free_info (struct kmem_cache *cache, -void *object,u8 tag) -{ -struct kasan_alloc_meta *alloc_meta; - - alloc_meta =kasan_get_alloc_meta (cache, object); -if (alloc_meta) -kasan_set_track (&alloc_meta->free_track[0 ],GFP_NOWAIT ); -} - -struct kasan_track *kasan_get_free_track (struct kmem_cache *cache, -void *object,u8 tag) -{ -struct kasan_alloc_meta *alloc_meta; - - alloc_meta =kasan_get_alloc_meta (cache, object); -if (!alloc_meta) -return NULL ; - -return &alloc_meta->free_track[0 ]; -} -#if IS_ENABLED (CONFIG_KASAN_KUNIT_TEST )void kasan_set_tagging_report_once (bool state)
mm/kasan/Makefile renamed from mm/kasan/Makefile~kasan-integrate-the-common-part-of-two-kasan-tag-based-modes +2 -2
@@ -37,5 +37,5 @@ CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUN obj-$(CONFIG_KASAN) := common.o report.o obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o -obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o -obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o +obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o tags.o report_tags.o +obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o tags.o report_tags.o
mm/kasan/report_hw_tags.c renamed from mm/kasan/report_hw_tags.c~kasan-integrate-the-common-part-of-two-kasan-tag-based-modes +0 -5
@@ -15,11 +15,6 @@#include "kasan.h" -const char *kasan_get_bug_type (struct kasan_access_info *info) -{ -return "invalid-access" ; -} -void *kasan_find_first_bad_addr (void *addr,size_t size) {return kasan_reset_tag (addr);
mm/kasan/report_sw_tags.c renamed from mm/kasan/report_sw_tags.c~kasan-integrate-the-common-part-of-two-kasan-tag-based-modes +0 -43
49 lines not on this page · raw message
mm/kasan/report_tags.c new file +51 -0
51 lines not on this page · raw message
mm/kasan/sw_tags.c renamed from mm/kasan/sw_tags.c~kasan-integrate-the-common-part-of-two-kasan-tag-based-modes +0 -41
44 lines not on this page · raw message
mm/kasan/tags.c new file +59 -0
59 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 147/192 kasan: add memory corruption identification support for hardware tag-based mode
In reply to Andrew MortonFrom: Kuan-Ying Lee <[email protected]>
Subject: kasan: add memory corruption identification support for hardware tag-based mode
Add memory corruption identification support for hardware tag-based mode. We store one old free pointer tag and free backtrace instead of five because hardware tag-based kasan only has 16 different tags.
If we store as many stacks as SW tag-based kasan does(5 stacks), there is high probability to find the same tag in the stacks when out-of-bound issues happened and we will mistake out-of-bound issue for use-after-free.
---
lib/Kconfig.kasan | 2 +- mm/kasan/kasan.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
lib/Kconfig.kasan renamed from lib/Kconfig.kasan~kasan-add-memory-corruption-identification-support-for-hardware-tag-based-mode +1 -1
@@ -169,7 +169,7 @@ config KASAN_STACK config KASAN_TAGS_IDENTIFY bool "Enable memory corruption identification" - depends on KASAN_SW_TAGS + depends on KASAN_SW_TAGS || KASAN_HW_TAGS help This option enables best-effort identification of bug type (use-after-free or out-of-bounds) at the cost of increased
mm/kasan/kasan.h renamed from mm/kasan/kasan.h~kasan-add-memory-corruption-identification-support-for-hardware-tag-based-mode +1 -1
@@ -153,7 +153,7 @@ struct kasan_track {depot_stack_handle_t stack; }; -#ifdef CONFIG_KASAN_TAGS_IDENTIFY +#if defined(CONFIG_KASAN_TAGS_IDENTIFY ) && defined(CONFIG_KASAN_SW_TAGS )#define KASAN_NR_FREE_STACKS 5#else #define KASAN_NR_FREE_STACKS 1
_
Andrew Morton
maintainerPATCH 148/192 mm: report which part of mem is being freed on initmem case
In reply to Andrew MortonFrom: Jungseung Lee <[email protected]>
Subject: mm: report which part of mem is being freed on initmem case
Add the details for figuring out which parts of the kernel image is being freed on initmem case.
Before:
Freeing unused kernel memory: 1024K
After:
Freeing unused kernel image (initmem) memory: 1024K
---
include/linux/mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-report-which-part-of-mem-is-being-freed-on-initmem-case +1 -1
@@ -2416,7 +2416,7 @@ static inline unsigned long free_initmemextern char __init_begin[], __init_end[];return free_reserved_area (&__init_begin, &__init_end, - poison,"unused kernel" ); + poison,"unused kernel image (initmem)" ); }static inline unsigned long get_num_physpages (void )
_
Andrew Morton
maintainerPATCH 149/192 mm/mmzone.h: simplify is_highmem_idx()
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: mm/mmzone.h: simplify is_highmem_idx()
There is a lot of historical ifdefery in is_highmem_idx() and its helper zone_movable_is_highmem() that was required because of two different paths for nodes and zones initialization that were selected at compile time.
Until commit 3f08a302f533 ("mm: remove CONFIG_HAVE_MEMBLOCK_NODE_MAP option") the movable_zone variable was only available for configurations that had CONFIG_HAVE_MEMBLOCK_NODE_MAP enabled so the test in zone_movable_is_highmem() used that variable only for such configurations. For other configurations the test checked if the index of ZONE_MOVABLE was greater by 1 than the index of ZONE_HIGMEM and then movable zone was considered a highmem zone. Needless to say, ZONE_MOVABLE - 1 equals ZONE_HIGHMEM by definition when CONFIG_HIGHMEM=y.
Commit 3f08a302f533 ("mm: remove CONFIG_HAVE_MEMBLOCK_NODE_MAP option") made movable_zone variable always available. Since this variable is set to ZONE_HIGHMEM if CONFIG_HIGHMEM is enabled and highmem zone is populated, it is enough to check whether
zone_idx == ZONE_MOVABLE && movable_zone == ZONE_HIGMEM
to test if zone index points to a highmem zone.
Remove zone_movable_is_highmem() that is not used anywhere except is_highmem_idx() and use the test above in is_highmem_idx() instead.
---
include/linux/mmzone.h | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-mmzoneh-simplify-is_highmem_idx +1 -12
@@ -982,22 +982,11 @@ static inline void zone_set_nid(struct zextern int movable_zone; -#ifdef CONFIG_HIGHMEM -static inline int zone_movable_is_highmem (void ) -{ -#ifdef CONFIG_NEED_MULTIPLE_NODES -return movable_zone ==ZONE_HIGHMEM ; -#else -return (ZONE_MOVABLE -1 ) ==ZONE_HIGHMEM ; -#endif -} -#endif -static inline int is_highmem_idx (enum zone_type idx) {#ifdef CONFIG_HIGHMEM return (idx ==ZONE_HIGHMEM || - (idx ==ZONE_MOVABLE &&zone_movable_is_highmem ())); + (idx ==ZONE_MOVABLE && movable_zone ==ZONE_HIGHMEM ));#else return 0 ;#endif
_
Andrew Morton
maintainerPATCH 150/192 mm: make __dump_page static
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm: make __dump_page static
Patch series "Constify struct page arguments".
While working on various solutions to the 32-bit struct page size regression, one of the problems I found was the networking stack expects to be able to pass const struct page pointers around, and the mm doesn't provide a lot of const-friendly functions to call. The root tangle of problems is that a lot of functions call VM_BUG_ON_PAGE(), which calls dump_page(), which calls a lot of functions which don't take a const struct page (but could be const).
This patch (of 6):
The only caller of __dump_page() now opencodes dump_page(), so remove it as an externally visible symbol.
---
include/linux/mmdebug.h | 3 +-- mm/debug.c | 2 +- mm/page_alloc.c | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-)
include/linux/mmdebug.h renamed from include/linux/mmdebug.h~mm-make-__dump_page-static +1 -2
@@ -9,8 +9,7 @@ struct page;struct vm_area_struct ;struct mm_struct ; -extern void dump_page (struct page *page,const char *reason); -extern void __dump_page (struct page *page,const char *reason); +void dump_page (struct page *page,const char *reason);void dump_vma (const struct vm_area_struct *vma);void dump_mm (const struct mm_struct *mm);
mm/debug.c renamed from mm/debug.c~mm-make-__dump_page-static +1 -1
@@ -42,7 +42,7 @@ const struct trace_print_flags vmaflag_n {0, NULL} }; -void __dump_page (struct page *page,const char *reason) +static void __dump_page (struct page *page,const char *reason) {struct page *head =compound_head (page);struct address_space *mapping;
mm/page_alloc.c renamed from mm/page_alloc.c~mm-make-__dump_page-static +1 -2
@@ -658,8 +658,7 @@ static void bad_page(struct page *page,pr_alert ("BUG: Bad page state in process %s pfn:%05lx\n" , current->comm,page_to_pfn (page)); -__dump_page (page, reason); -dump_page_owner (page); +dump_page (page, reason);print_modules ();dump_stack ();
_
Andrew Morton
maintainerPATCH 151/192 mm/page_alloc: bail out on fatal signal during reclaim/compaction retry attempt
In reply to Andrew MortonFrom: Aaron Tomlin <[email protected]>
Subject: mm/page_alloc: bail out on fatal signal during reclaim/compaction retry attempt
A customer experienced a low-memory situation and decided to issue a SIGKILL (i.e. a fatal signal). Instead of promptly terminating as one would expect, the aforementioned task remained unresponsive.
Further investigation indicated that the task was "stuck" in the reclaim/compaction retry loop. Now, it does not make sense to retry compaction when a fatal signal is pending.
In the context of try_to_compact_pages(), indeed COMPACT_SKIPPED can be returned; albeit, not every zone, on the zone list, would be considered in the case a fatal signal is found to be pending. Yet, in should_compact_retry(), given the last known compaction result, each zone, on the zone list, can be considered/or checked (see compaction_zonelist_suitable()). For example, if a zone was found to succeed, then reclaim/compaction would be tried again (notwithstanding the above).
This patch ensures that compaction is not needlessly retried irrespective of the last known compaction result e.g. if it was skipped, in the unlikely case a fatal signal is found pending. So, OOM is at least attempted.
---
mm/page_alloc.c | 3 +++ 1 file changed, 3 insertions(+)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-bail-out-on-fatal-signal-during-reclaim-compaction-retry-attempt +3 -0
@@ -4251,6 +4251,9 @@ should_compact_retry(struct alloc_contexif (!order)return false; +if (fatal_signal_pending (current)) +return false; +if (compaction_made_progress (compact_result)) (*compaction_retries)++;
_
Andrew Morton
maintainerPATCH 152/192 mm/debug: factor PagePoisoned out of __dump_page
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm/debug: factor PagePoisoned out of __dump_page
Move the PagePoisoned test into dump_page(). Skip the hex print for poisoned pages -- we know they're full of ffffffff. Move the reason printing from __dump_page() to dump_page().
---
mm/debug.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-)
mm/debug.c renamed from mm/debug.c~mm-debug-factor-pagepoisoned-out-of-__dump_page +7 -18
@@ -42,11 +42,10 @@ const struct trace_print_flags vmaflag_n {0, NULL} }; -static void __dump_page (struct page *page,const char *reason) +static void __dump_page (struct page *page) {struct page *head =compound_head (page);struct address_space *mapping; -bool page_poisoned =PagePoisoned (page);bool compound =PageCompound (page);/* * Accessing the pageblock without the zone lock. It could change to
@@ -58,16 +57,6 @@ static void __dump_page(struct page *pagint mapcount;char *type ="" ; -/* -* If struct page is poisoned don't access Page*() functions as that -* leads to recursive loop. Page*() check for poisoned pages, and calls -* dump_page() when detected. -*/ -if (page_poisoned) { -pr_warn ("page:%px is uninitialized and poisoned" , page); - goto hex_only; - } -if (page < head || (page >= head +MAX_ORDER_NR_PAGES )) {/* * Corrupt page, so we cannot call page_mapping. Instead, do a
@@ -173,8 +162,6 @@ out_mapping:pr_warn ("%sflags: %#lx(%pGp)%s\n" , type, head->flags, &head->flags, page_cma ?" CMA" :"" ); - -hex_only:print_hex_dump (KERN_WARNING ,"raw: " ,DUMP_PREFIX_NONE ,32 ,sizeof (unsigned long ), page,sizeof (struct page ), false);
@@ -182,14 +169,16 @@ hex_only:print_hex_dump (KERN_WARNING ,"head: " ,DUMP_PREFIX_NONE ,32 ,sizeof (unsigned long ), head,sizeof (struct page ), false); - -if (reason) -pr_warn ("page dumped because: %s\n" , reason); }void dump_page (struct page *page,const char *reason) { -__dump_page (page, reason); +if (PagePoisoned (page)) +pr_warn ("page:%p is uninitialized and poisoned" , page); +else +__dump_page (page); +if (reason) +pr_warn ("page dumped because: %s\n" , reason);dump_page_owner (page); }EXPORT_SYMBOL (dump_page);
_
Andrew Morton
maintainerPATCH 153/192 mm/page_owner: constify dump_page_owner
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm/page_owner: constify dump_page_owner
dump_page_owner() only uses struct page to find the page_ext, and lookup_page_ext() already takes a const argument.
---
include/linux/page_owner.h | 6 +++--- mm/page_owner.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/page_owner.h renamed from include/linux/page_owner.h~mm-page_owner-constify-dump_page_owner +3 -3
@@ -14,7 +14,7 @@ extern void __set_page_owner(struct pageextern void __split_page_owner (struct page *page,unsigned int nr);extern void __copy_page_owner (struct page *oldpage,struct page *newpage);extern void __set_page_owner_migrate_reason (struct page *page,int reason); -extern void __dump_page_owner (struct page *page); +extern void __dump_page_owner (const struct page *page);extern void pagetypeinfo_showmixedcount_print (struct seq_file *m,pg_data_t *pgdat,struct zone *zone);
@@ -46,7 +46,7 @@ static inline void set_page_owner_migratif (static_branch_unlikely (&page_owner_inited))__set_page_owner_migrate_reason (page, reason); } -static inline void dump_page_owner (struct page *page) +static inline void dump_page_owner (const struct page *page) {if (static_branch_unlikely (&page_owner_inited))__dump_page_owner (page);
@@ -69,7 +69,7 @@ static inline void copy_page_owner(strucstatic inline void set_page_owner_migrate_reason (struct page *page,int reason) { } -static inline void dump_page_owner (struct page *page) +static inline void dump_page_owner (const struct page *page) { }#endif /* CONFIG_PAGE_OWNER */
mm/page_owner.c renamed from mm/page_owner.c~mm-page_owner-constify-dump_page_owner +1 -1
@@ -392,7 +392,7 @@ err:return -ENOMEM ; } -void __dump_page_owner (struct page *page) +void __dump_page_owner (const struct page *page) {struct page_ext *page_ext =lookup_page_ext (page);struct page_owner *page_owner;
_
Andrew Morton
maintainerPATCH 154/192 mm: make compound_head const-preserving
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm: make compound_head const-preserving
If you pass a const pointer to compound_head(), you get a const pointer back; if you pass a mutable pointer, you get a mutable pointer back. Also remove an unnecessary forward definition of struct page; we're about to dereference page->compound_head, so it must already have been defined.
---
include/linux/page-flags.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
include/linux/page-flags.h renamed from include/linux/page-flags.h~mm-make-compound_head-const-preserving +5 -5
@@ -177,17 +177,17 @@ enum pageflags {#ifndef __GENERATING_BOUNDS_H -struct page ;/* forward declaration */ - -static inline struct page *compound_head (struct page *page) +static inline unsigned long _compound_head (const struct page *page) {unsigned long head =READ_ONCE (page->compound_head);if (unlikely (head &1 )) -return (struct page *) (head -1 ); -return page; +return head -1 ; +return (unsigned long )page; } +#define compound_head (page) ((typeof(page))_compound_head(page)) + static __always_inline int PageTail(struct page *page) {return READ_ONCE (page->compound_head) &1 ;
_
Andrew Morton
maintainerPATCH 155/192 mm: constify get_pfnblock_flags_mask and get_pfnblock_migratetype
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm: constify get_pfnblock_flags_mask and get_pfnblock_migratetype
The struct page is not modified by these routines, so it can be marked const.
---
include/linux/pageblock-flags.h | 2 +- mm/page_alloc.c | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-)
include/linux/pageblock-flags.h renamed from include/linux/pageblock-flags.h~mm-constify-get_pfnblock_flags_mask-and-get_pfnblock_migratetype +1 -1
@@ -54,7 +54,7 @@ extern unsigned int pageblock_order;/* Forward declaration */ struct page ; -unsigned long get_pfnblock_flags_mask (struct page *page, +unsigned long get_pfnblock_flags_mask (const struct page *page,unsigned long pfn,unsigned long mask);
mm/page_alloc.c renamed from mm/page_alloc.c~mm-constify-get_pfnblock_flags_mask-and-get_pfnblock_migratetype +7 -6
@@ -474,7 +474,7 @@ static inline bool defer_init(int nid, u#endif /* Return a pointer to the bitmap storing bits affecting a block of pages */ -static inline unsigned long *get_pageblock_bitmap (struct page *page, +static inline unsigned long *get_pageblock_bitmap (const struct page *page,unsigned long pfn) {#ifdef CONFIG_SPARSEMEM
@@ -484,7 +484,7 @@ static inline unsigned long *get_pageblo#endif /* CONFIG_SPARSEMEM */ } -static inline int pfn_to_bitidx (struct page *page,unsigned long pfn) +static inline int pfn_to_bitidx (const struct page *page,unsigned long pfn) {#ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION -1 );
@@ -495,7 +495,7 @@ static inline int pfn_to_bitidx(struct p }static __always_inline -unsigned long __get_pfnblock_flags_mask (struct page *page, +unsigned long __get_pfnblock_flags_mask (const struct page *page,unsigned long pfn,unsigned long mask) {
@@ -520,13 +520,14 @@ unsigned long __get_pfnblock_flags_mask(* * Return: pageblock_bits flags */ -unsigned long get_pfnblock_flags_mask (struct page *page,unsigned long pfn, -unsigned long mask) +unsigned long get_pfnblock_flags_mask (const struct page *page, +unsigned long pfn,unsigned long mask) {return __get_pfnblock_flags_mask (page, pfn, mask); } -static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) +static __always_inline int get_pfnblock_migratetype(const struct page *page, +unsigned long pfn) {return __get_pfnblock_flags_mask (page, pfn,MIGRATETYPE_MASK ); }
_
Andrew Morton
maintainerPATCH 156/192 mm: constify page_count and page_ref_count
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm: constify page_count and page_ref_count
Now that compound_head() accepts a const struct page pointer, these two functions can be marked as not modifying the page pointer they are passed.
---
include/linux/page_ref.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
include/linux/page_ref.h renamed from include/linux/page_ref.h~mm-constify-page_count-and-page_ref_count +2 -2
@@ -62,12 +62,12 @@ static inline void __page_ref_unfreeze(s#endif -static inline int page_ref_count (struct page *page) +static inline int page_ref_count (const struct page *page) {return atomic_read (&page->_refcount); } -static inline int page_count (struct page *page) +static inline int page_count (const struct page *page) {return atomic_read (&compound_head (page)->_refcount); }
_
Andrew Morton
maintainerPATCH 157/192 mm: optimise nth_page for contiguous memmap
In reply to Andrew MortonFrom: "Matthew Wilcox (Oracle)" <[email protected]>
Subject: mm: optimise nth_page for contiguous memmap
If the memmap is virtually contiguous (either because we're using a virtually mapped memmap or because we don't support a discontig memmap at all), then we can implement nth_page() by simple addition. Contrary to popular belief, the compiler is not able to optimise this itself for a vmemmap configuration. This reduces one example user (sg.c) by four instructions:
struct page *page = nth_page(rsv_schp->pages[k], offset >> PAGE_SHIFT);
before:
49 8b 45 70 mov 0x70(%r13),%rax
48 63 c9 movslq %ecx,%rcx
48 c1 eb 0c shr $0xc,%rbx
48 8b 04 c8 mov (%rax,%rcx,8),%rax
48 2b 05 00 00 00 00 sub 0x0(%rip),%rax
R_X86_64_PC32 vmemmap_base-0x4
48 c1 f8 06 sar $0x6,%rax
48 01 d8 add %rbx,%rax
48 c1 e0 06 shl $0x6,%rax
48 03 05 00 00 00 00 add 0x0(%rip),%rax
R_X86_64_PC32 vmemmap_base-0x4after:
49 8b 45 70 mov 0x70(%r13),%rax 48 63 c9 movslq %ecx,%rcx 48 c1 eb 0c shr $0xc,%rbx 48 c1 e3 06 shl $0x6,%rbx 48 03 1c c8 add (%rax,%rcx,8),%rbx
---
include/linux/mm.h | 4 ++++ 1 file changed, 4 insertions(+)
include/linux/mm.h renamed from include/linux/mm.h~mm-optimise-nth_page-for-contiguous-memmap +4 -0
@@ -234,7 +234,11 @@ int overcommit_policy_handler(struct ctlint __add_to_page_cache_locked (struct page *page,struct address_space *mapping,pgoff_t index,gfp_t gfp,void **shadowp); +#if defined(CONFIG_SPARSEMEM ) && !defined(CONFIG_SPARSEMEM_VMEMMAP )#define nth_page (page,n) pfn_to_page(page_to_pfn((page)) + (n)) +#else +#define nth_page (page,n) ((page) + (n)) +#endif /* to align the pointer to the (next) page boundary */ #define PAGE_ALIGN (addr) ALIGN(addr, PAGE_SIZE)
_
Andrew Morton
maintainerPATCH 158/192 mm/page_alloc: switch to pr_debug
In reply to Andrew MortonFrom: Heiner Kallweit <[email protected]>
Subject: mm/page_alloc: switch to pr_debug
Having such debug messages in the dmesg log may confuse users. Therefore restrict debug output to cases where DEBUG is defined or dynamic debugging is enabled for the respective code piece.
---
mm/page_alloc.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-switch-to-pr_debug +6 -10
@@ -6770,9 +6770,8 @@ static __meminit void zone_pcp_init(stru zone->pageset_batch =BOOT_PAGESET_BATCH ;if (populated_zone (zone)) -printk (KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n" , - zone->name, zone->present_pages, -zone_batchsize (zone)); +pr_debug (" %s zone: %lu pages, LIFO batch:%u\n" , zone->name, + zone->present_pages,zone_batchsize (zone)); } void __meminit init_currently_empty_zone(struct zone *zone,
@@ -7042,8 +7041,7 @@ static void __init calculate_node_totalp pgdat->node_spanned_pages = totalpages; pgdat->node_present_pages = realtotalpages; -printk (KERN_DEBUG "On node %d totalpages: %lu\n" , pgdat->node_id, - realtotalpages); +pr_debug ("On node %d totalpages: %lu\n" , pgdat->node_id, realtotalpages); }#ifndef CONFIG_SPARSEMEM
@@ -7243,9 +7241,8 @@ static void __init free_area_init_core(sif (freesize >= memmap_pages) { freesize -= memmap_pages;if (memmap_pages) -printk (KERN_DEBUG -" %s zone: %lu pages used for memmap\n" , - zone_names[j], memmap_pages); +pr_debug (" %s zone: %lu pages used for memmap\n" , + zone_names[j], memmap_pages); }else pr_warn (" %s zone: %lu pages exceeds freesize %lu\n" , zone_names[j], memmap_pages, freesize);
@@ -7254,8 +7251,7 @@ static void __init free_area_init_core(s/* Account for reserved pages */ if (j ==0 && freesize > dma_reserve) { freesize -= dma_reserve; -printk (KERN_DEBUG " %s zone: %lu pages reserved\n" , - zone_names[0 ], dma_reserve); +pr_debug (" %s zone: %lu pages reserved\n" , zone_names[0 ], dma_reserve); }if (!is_highmem_idx (j))
_
Andrew Morton
maintainerPATCH 159/192 kbuild: skip per-CPU BTF generation for pahole v1.18-v1.21
In reply to Andrew MortonFrom: Andrii Nakryiko <[email protected]>
Subject: kbuild: skip per-CPU BTF generation for pahole v1.18-v1.21
Commit "mm/page_alloc: convert per-cpu list protection to local_lock" will introduce a zero-sized per-CPU variable, which causes pahole to generate invalid BTF. Only pahole versions 1.18 through 1.21 are impacted, as before 1.18 pahole doesn't know anything about per-CPU variables, and 1.22 contains the proper fix for the issue.
Luckily, pahole 1.18 got --skip_encoding_btf_vars option disabling BTF generation for per-CPU variables in anticipation of some unanticipated problems. So use this escape hatch to disable per-CPU var BTF info on those problematic pahole versions. Users relying on availability of per-CPU var BTFs would need to upgrade to pahole 1.22+, but everyone won't notice any regressions.
---
scripts/link-vmlinux.sh | 4 ++++ 1 file changed, 4 insertions(+)
scripts/link-vmlinux.sh renamed from scripts/link-vmlinux.sh~kbuild-skip-per-cpu-btf-generation-for-pahole-v118-v121 +4 -0
@@ -235,6 +235,10 @@ gen_btf() vmlinux_link ${1} + if [ "${pahole_ver}" -ge "118" ] && [ "${pahole_ver}" -le "121" ]; then + # pahole 1.18 through 1.21 can't handle zero-sized per-CPU vars + extra_paholeopt="${extra_paholeopt} --skip_encoding_btf_vars" + fi if [ "${pahole_ver}" -ge "121" ]; then extra_paholeopt="${extra_paholeopt} --btf_gen_floats" fi
_
Andrew Morton
maintainerPATCH 160/192 mm/page_alloc: split per cpu page lists and zone stats
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: split per cpu page lists and zone stats
The PCP (per-cpu page allocator in page_alloc.c) shares locking requirements with vmstat and the zone lock which is inconvenient and causes some issues. For example, the PCP list and vmstat share the same per-cpu space meaning that it's possible that vmstat updates dirty cache lines holding per-cpu lists across CPUs unless padding is used. Second, PREEMPT_RT does not want to disable IRQs for too long in the page allocator.
This series splits the locking requirements and uses locks types more suitable for PREEMPT_RT, reduces the time when special locking is required for stats and reduces the time when IRQs need to be disabled on !PREEMPT_RT kernels.
Why local_lock? PREEMPT_RT considers the following sequence to be unsafe as documented in Documentation/locking/locktypes.rst
local_irq_disable(); spin_lock(&lock);
The pcp allocator has this sequence for rmqueue_pcplist (local_irq_save) -> __rmqueue_pcplist -> rmqueue_bulk (spin_lock). While it's possible to separate this out, it generally means there are points where we enable IRQs and reenable them again immediately. To prevent a migration and the per-cpu pointer going stale, migrate_disable is also needed. That is a custom lock that is similar, but worse, than local_lock. Furthermore, on PREEMPT_RT, it's undesirable to leave IRQs disabled for too long. By converting to local_lock which disables migration on PREEMPT_RT, the locking requirements can be separated and start moving the protections for PCP, stats and the zone lock to PREEMPT_RT-safe equivalent locking. As a bonus, local_lock also means that PROVE_LOCKING does something useful.
After that, it's obvious that zone_statistics incurs too much overhead and leaves IRQs disabled for longer than necessary on !PREEMPT_RT kernels. zone_statistics uses perfectly accurate counters requiring IRQs be disabled for parallel RMW sequences when inaccurate ones like vm_events would do. The series makes the NUMA statistics (NUMA_HIT and friends) inaccurate counters that then require no special protection on !PREEMPT_RT.
The bulk page allocator can then do stat updates in bulk with IRQs enabled which should improve the efficiency. Technically, this could have been done without the local_lock and vmstat conversion work and the order simply reflects the timing of when different series were implemented.
Finally, there are places where we conflate IRQs being disabled for the PCP with the IRQ-safe zone spinlock. The remainder of the series reduces the scope of what is protected by disabled IRQs on !PREEMPT_RT kernels. By the end of the series, page_alloc.c does not call local_irq_save so the locking scope is a bit clearer. The one exception is that modifying NR_FREE_PAGES still happens in places where it's known the IRQs are disabled as it's harmless for PREEMPT_RT and would be expensive to split the locking there.
No performance data is included because despite the overhead of the stats, it's within the noise for most workloads on !PREEMPT_RT. However, Jesper Dangaard Brouer ran a page allocation microbenchmark on a E5-1650 v4 @ 3.60GHz CPU on the first version of this series. Focusing on the array variant of the bulk page allocator reveals the following.
(CPU: Intel(R) Xeon(R) CPU E5-1650 v4 @ 3.60GHz) ARRAY variant: time_bulk_page_alloc_free_array: step=bulk size
Baseline Patched 1 56.383 54.225 (+3.83%) 2 40.047 35.492 (+11.38%) 3 37.339 32.643 (+12.58%) 4 35.578 30.992 (+12.89%) 8 33.592 29.606 (+11.87%) 16 32.362 28.532 (+11.85%) 32 31.476 27.728 (+11.91%) 64 30.633 27.252 (+11.04%) 128 30.596 27.090 (+11.46%)
While this is a positive outcome, the series is more likely to be interesting to the RT people in terms of getting parts of the PREEMPT_RT tree into mainline.
This patch (of 9):
The per-cpu page allocator lists and the per-cpu vmstat deltas are stored in the same struct per_cpu_pages even though vmstats have no direct impact on the per-cpu page lists. This is inconsistent because the vmstats for a node are stored on a dedicated structure. The bigger issue is that the per_cpu_pages structure is not cache-aligned and stat updates either cache conflict with adjacent per-cpu lists incurring a runtime cost or padding is required incurring a memory cost.
This patch splits the per-cpu pagelists and the vmstat deltas into separate structures. It's mostly a mechanical conversion but some variable renaming is done to clearly distinguish the per-cpu pages structure (pcp) from the vmstats (pzstats).
Superficially, this appears to increase the size of the per_cpu_pages structure but the movement of expire fills a structure hole so there is no impact overall.
---
include/linux/mmzone.h | 18 +++---- include/linux/vmstat.h | 8 +-- mm/page_alloc.c | 85 ++++++++++++++++++--------------- mm/vmstat.c | 98 ++++++++++++++++++++------------------- 4 files changed, 113 insertions(+), 96 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-split-per-cpu-page-lists-and-zone-stats +10 -8
@@ -341,20 +341,21 @@ struct per_cpu_pages {int count;/* number of pages in the list */ int high;/* high watermark, emptying needed */ int batch;/* chunk size for buddy add/remove */ +#ifdef CONFIG_NUMA +int expire;/* When 0, remote pagesets are drained */ +#endif /* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[MIGRATE_PCPTYPES ]; }; -struct per_cpu_pageset { -struct per_cpu_pages pcp; -#ifdef CONFIG_NUMA -s8 expire; -u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS ]; -#endif +struct per_cpu_zonestat {#ifdef CONFIG_SMP -s8 stat_threshold;s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS ]; +s8 stat_threshold; +#endif +#ifdef CONFIG_NUMA +u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS ];#endif };
@@ -484,7 +485,8 @@ struct zone {int node;#endif struct pglist_data *zone_pgdat; -struct per_cpu_pageset __percpu *pageset; +struct per_cpu_pages __percpu *per_cpu_pageset; +struct per_cpu_zonestat __percpu *per_cpu_zonestats;/* * the high and batch values are copied to individual pagesets for * faster access
include/linux/vmstat.h renamed from include/linux/vmstat.h~mm-page_alloc-split-per-cpu-page-lists-and-zone-stats +4 -4
@@ -163,7 +163,7 @@ static inline unsigned long zone_numa_stint cpu;for_each_online_cpu (cpu) - x +=per_cpu_ptr (zone->pageset, cpu)->vm_numa_stat_diff[item]; + x +=per_cpu_ptr (zone->per_cpu_zonestats, cpu)->vm_numa_stat_diff[item];return x; }
@@ -236,7 +236,7 @@ static inline unsigned long zone_page_st#ifdef CONFIG_SMP int cpu;for_each_online_cpu (cpu) - x +=per_cpu_ptr (zone->pageset, cpu)->vm_stat_diff[item]; + x +=per_cpu_ptr (zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];if (x <0 ) x =0 ;
@@ -291,7 +291,7 @@ struct ctl_table;int vmstat_refresh (struct ctl_table *,int write,void *buffer,size_t *lenp,loff_t *ppos); -void drain_zonestat (struct zone *zone,struct per_cpu_pageset *); +void drain_zonestat (struct zone *zone,struct per_cpu_zonestat *);int calculate_pressure_threshold (struct zone *zone);int calculate_normal_threshold (struct zone *zone);
@@ -399,7 +399,7 @@ static inline void cpu_vm_stats_fold(intstatic inline void quiet_vmstat (void ) { }static inline void drain_zonestat (struct zone *zone, -struct per_cpu_pageset *pset) { } +struct per_cpu_zonestat *pzstats) { }#endif /* CONFIG_SMP */ static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-split-per-cpu-page-lists-and-zone-stats +47 -38
212 lines not on this page · raw message
mm/vmstat.c renamed from mm/vmstat.c~mm-page_alloc-split-per-cpu-page-lists-and-zone-stats +52 -46
252 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 161/192 mm/page_alloc: convert per-cpu list protection to local_lock
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: convert per-cpu list protection to local_lock
There is a lack of clarity of what exactly local_irq_save/local_irq_restore protects in page_alloc.c . It conflates the protection of per-cpu page allocation structures with per-cpu vmstat deltas.
This patch protects the PCP structure using local_lock which for most configurations is identical to IRQ enabling/disabling. The scope of the lock is still wider than it should be but this is decreased later.
It is possible for the local_lock to be embedded safely within struct per_cpu_pages but it adds complexity to free_unref_page_list.
---
include/linux/mmzone.h | 2 + lib/Kconfig.debug | 3 + mm/page_alloc.c | 61 +++++++++++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 15 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-convert-per-cpu-list-protection-to-local_lock +2 -0
@@ -20,6 +20,7 @@#include <linux/atomic.h> #include <linux/mm_types.h> #include <linux/page-flags.h> +#include <linux/local_lock.h> #include <asm/page.h> /* Free memory management - zoned buddy allocator. */
@@ -337,6 +338,7 @@ enum zone_watermarks {#define high_wmark_pages (z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)#define wmark_pages (z, i) (z->_watermark[i] + z->watermark_boost) +/* Fields and list protected by pagesets local_lock in page_alloc.c */ struct per_cpu_pages {int count;/* number of pages in the list */ int high;/* high watermark, emptying needed */
lib/Kconfig.debug renamed from lib/Kconfig.debug~mm-page_alloc-convert-per-cpu-list-protection-to-local_lock +3 -0
@@ -313,6 +313,9 @@ config DEBUG_INFO_BTF config PAHOLE_HAS_SPLIT_BTF def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") +config PAHOLE_HAS_ZEROSIZE_PERCPU_SUPPORT + def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "122") + config DEBUG_INFO_BTF_MODULES def_bool y depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-convert-per-cpu-list-protection-to-local_lock +46 -15
156 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 162/192 mm/vmstat: convert NUMA statistics to basic NUMA counters
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/vmstat: convert NUMA statistics to basic NUMA counters
NUMA statistics are maintained on the zone level for hits, misses, foreign etc but nothing relies on them being perfectly accurate for functional correctness. The counters are used by userspace to get a general overview of a workloads NUMA behaviour but the page allocator incurs a high cost to maintain perfect accuracy similar to what is required for a vmstat like NR_FREE_PAGES. There even is a sysctl vm.numa_stat to allow userspace to turn off the collection of NUMA statistics like NUMA_HIT.
This patch converts NUMA_HIT and friends to be NUMA events with similar accuracy to VM events. There is a possibility that slight errors will be introduced but the overall trend as seen by userspace will be similar. The counters are no longer updated from vmstat_refresh context as it is unnecessary overhead for counters that may never be read by userspace. Note that counters could be maintained at the node level to save space but it would have a user-visible impact due to /proc/zoneinfo.
---
drivers/base/node.c | 18 ++-- include/linux/mmzone.h | 13 ++ include/linux/vmstat.h | 43 ++++----- mm/mempolicy.c | 2 mm/page_alloc.c | 12 +- mm/vmstat.c | 175 +++++++++++++++------------------------ 6 files changed, 114 insertions(+), 149 deletions(-)
drivers/base/node.c renamed from drivers/base/node.c~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +10 -8
@@ -482,6 +482,7 @@ static DEVICE_ATTR(meminfo, 0444, node_rstatic ssize_t node_read_numastat (struct device *dev,struct device_attribute *attr,char *buf) { +fold_vm_numa_events ();return sysfs_emit (buf,"numa_hit %lu\n" "numa_miss %lu\n"
@@ -489,12 +490,12 @@ static ssize_t node_read_numastat(struct"interleave_hit %lu\n" "local_node %lu\n" "other_node %lu\n" , -sum_zone_numa_state (dev->id,NUMA_HIT ), -sum_zone_numa_state (dev->id,NUMA_MISS ), -sum_zone_numa_state (dev->id,NUMA_FOREIGN ), -sum_zone_numa_state (dev->id,NUMA_INTERLEAVE_HIT ), -sum_zone_numa_state (dev->id,NUMA_LOCAL ), - sum_zone_numa_state(dev->id, NUMA_OTHER)); +sum_zone_numa_event_state (dev->id,NUMA_HIT ), +sum_zone_numa_event_state (dev->id,NUMA_MISS ), +sum_zone_numa_event_state (dev->id,NUMA_FOREIGN ), +sum_zone_numa_event_state (dev->id,NUMA_INTERLEAVE_HIT ), +sum_zone_numa_event_state (dev->id,NUMA_LOCAL ), + sum_zone_numa_event_state(dev->id, NUMA_OTHER)); } static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
@@ -512,10 +513,11 @@ static ssize_t node_read_vmstat(struct d sum_zone_node_page_state(nid, i));#ifdef CONFIG_NUMA -for (i =0 ; i <NR_VM_NUMA_STAT_ITEMS ; i++) +fold_vm_numa_events (); +for (i =0 ; i <NR_VM_NUMA_EVENT_ITEMS ; i++) len +=sysfs_emit_at (buf, len,"%s %lu\n" ,numa_stat_name (i), -sum_zone_numa_state (nid, i)); +sum_zone_numa_event_state (nid, i));#endif for (i =0 ; i <NR_VM_NODE_STAT_ITEMS ; i++) {
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +9 -4
@@ -135,10 +135,10 @@ enum numa_stat_item {NUMA_INTERLEAVE_HIT ,/* interleaver preferred this zone */ NUMA_LOCAL ,/* allocation from local node */ NUMA_OTHER ,/* allocation from other node */ -NR_VM_NUMA_STAT_ITEMS +NR_VM_NUMA_EVENT_ITEMS };#else -#define NR_VM_NUMA_STAT_ITEMS 0 +#define NR_VM_NUMA_EVENT_ITEMS 0#endif enum zone_stat_item {
@@ -357,7 +357,12 @@ struct per_cpu_zonestat {s8 stat_threshold;#endif #ifdef CONFIG_NUMA -u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS ]; +/* +* Low priority inaccurate counters that are only folded +* on demand. Use a large type to avoid the overhead of +* folding during refresh_cpu_vm_stats. +*/ +unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS ];#endif };
@@ -623,7 +628,7 @@ struct zone {ZONE_PADDING (_pad3_)/* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; -atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS ]; +atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS ]; }____cacheline_internodealigned_in_smp ;enum pgdat_flags {
include/linux/vmstat.h renamed from include/linux/vmstat.h~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +20 -23
93 lines not on this page · raw message
mm/mempolicy.c renamed from mm/mempolicy.c~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +1 -1
@@ -2150,7 +2150,7 @@ static struct page *alloc_page_interleavreturn page;if (page &&page_to_nid (page) == nid) {preempt_disable (); -__inc_numa_state (page_zone (page),NUMA_INTERLEAVE_HIT ); +__count_numa_event (page_zone (page),NUMA_INTERLEAVE_HIT );preempt_enable (); }return page;
mm/page_alloc.c renamed from mm/page_alloc.c~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +6 -6
@@ -3480,12 +3480,12 @@ static inline void zone_statistics(struc local_stat =NUMA_OTHER ;if (zone_to_nid (z) ==zone_to_nid (preferred_zone)) -__inc_numa_state (z,NUMA_HIT ); +__count_numa_event (z,NUMA_HIT );else { -__inc_numa_state (z,NUMA_MISS ); -__inc_numa_state (preferred_zone,NUMA_FOREIGN ); +__count_numa_event (z,NUMA_MISS ); +__count_numa_event (preferred_zone,NUMA_FOREIGN ); } -__inc_numa_state (z, local_stat); +__count_numa_event (z, local_stat);#endif }
@@ -6785,8 +6785,8 @@ void __init setup_per_cpu_pageset(void)*/ for_each_possible_cpu (cpu) {struct per_cpu_zonestat *pzstats = &per_cpu (boot_zonestats, cpu); -memset (pzstats->vm_numa_stat_diff,0 , -sizeof (pzstats->vm_numa_stat_diff)); +memset (pzstats->vm_numa_event,0 , +sizeof (pzstats->vm_numa_event)); }#endif
mm/vmstat.c renamed from mm/vmstat.c~mm-vmstat-convert-numa-statistics-to-basic-numa-counters +68 -107
337 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 163/192 mm/vmstat: inline NUMA event counter updates
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/vmstat: inline NUMA event counter updates
__count_numa_event is small enough to be treated similarly to __count_vm_event so inline it.
---
include/linux/vmstat.h | 10 +++++++++- mm/vmstat.c | 9 --------- 2 files changed, 9 insertions(+), 10 deletions(-)
include/linux/vmstat.h renamed from include/linux/vmstat.h~mm-vmstat-inline-numa-event-counter-updates +9 -1
@@ -238,7 +238,15 @@ static inline unsigned long zone_page_st }#ifdef CONFIG_NUMA -extern void __count_numa_event (struct zone *zone,enum numa_stat_item item); +/* See __count_vm_event comment on why raw_cpu_inc is used. */ +static inline void +__count_numa_event (struct zone *zone,enum numa_stat_item item) +{ +struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; + +raw_cpu_inc (pzstats->vm_numa_event[item]); +} +extern unsigned long sum_zone_node_page_state (int node,enum zone_stat_item item);extern unsigned long sum_zone_numa_event_state (int node,enum numa_stat_item item);
mm/vmstat.c renamed from mm/vmstat.c~mm-vmstat-inline-numa-event-counter-updates +0 -9
@@ -938,15 +938,6 @@ void drain_zonestat(struct zone *zone, s#endif #ifdef CONFIG_NUMA -/* See __count_vm_event comment on why raw_cpu_inc is used. */ -void __count_numa_event (struct zone *zone, -enum numa_stat_item item) -{ -struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; - -raw_cpu_inc (pzstats->vm_numa_event[item]); -} -/* * Determine the per node value of a stat item. This function * is called frequently in a NUMA machine, so try to be as
_
Andrew Morton
maintainerPATCH 164/192 mm/page_alloc: batch the accounting updates in the bulk allocator
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: batch the accounting updates in the bulk allocator
Now that the zone_statistics are simple counters that do not require special protection, the bulk allocator accounting updates can be batch updated without adding too much complexity with protected RMW updates or using xchg.
---
include/linux/vmstat.h | 8 ++++++++ mm/page_alloc.c | 30 +++++++++++++----------------- 2 files changed, 21 insertions(+), 17 deletions(-)
include/linux/vmstat.h renamed from include/linux/vmstat.h~mm-page_alloc-batch-the-accounting-updates-in-the-bulk-allocator +8 -0
@@ -247,6 +247,14 @@ __count_numa_event(struct zone *zone, enraw_cpu_inc (pzstats->vm_numa_event[item]); } +static inline void +__count_numa_events (struct zone *zone,enum numa_stat_item item,long delta) +{ +struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; + +raw_cpu_add (pzstats->vm_numa_event[item], delta); +} +extern unsigned long sum_zone_node_page_state (int node,enum zone_stat_item item);extern unsigned long sum_zone_numa_event_state (int node,enum numa_stat_item item);
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-batch-the-accounting-updates-in-the-bulk-allocator +13 -17
@@ -3467,7 +3467,8 @@ void __putback_isolated_page(struct page* * Must be called with interrupts disabled. */ -static inline void zone_statistics (struct zone *preferred_zone,struct zone *z) +static inline void zone_statistics (struct zone *preferred_zone,struct zone *z, +long nr_account) {#ifdef CONFIG_NUMA enum numa_stat_item local_stat =NUMA_LOCAL ;
@@ -3480,12 +3481,12 @@ static inline void zone_statistics(struc local_stat =NUMA_OTHER ;if (zone_to_nid (z) ==zone_to_nid (preferred_zone)) -__count_numa_event (z,NUMA_HIT ); +__count_numa_events (z,NUMA_HIT , nr_account);else { -__count_numa_event (z,NUMA_MISS ); -__count_numa_event (preferred_zone,NUMA_FOREIGN ); +__count_numa_events (z,NUMA_MISS , nr_account); +__count_numa_events (preferred_zone,NUMA_FOREIGN , nr_account); } -__count_numa_event (z, local_stat); +__count_numa_events (z, local_stat, nr_account);#endif }
@@ -3531,7 +3532,7 @@ static struct page *rmqueue_pcplist(stru page =__rmqueue_pcplist (zone, migratetype, alloc_flags, pcp, list);if (page) {__count_zid_vm_events (PGALLOC ,page_zonenum (page),1 ); -zone_statistics (preferred_zone, zone); +zone_statistics (preferred_zone, zone,1 ); }local_unlock_irqrestore (&pagesets.lock, flags);return page;
@@ -3592,7 +3593,7 @@ struct page *rmqueue(struct zone *prefer get_pcppage_migratetype(page));__count_zid_vm_events (PGALLOC ,page_zonenum (page),1 << order); -zone_statistics (preferred_zone, zone); +zone_statistics (preferred_zone, zone,1 );local_irq_restore (flags); out:
@@ -5077,7 +5078,7 @@ unsigned long __alloc_pages_bulk(gfp_t gstruct alloc_context ac;gfp_t alloc_gfp;unsigned int alloc_flags =ALLOC_WMARK_LOW ; -int nr_populated =0 ; +int nr_populated =0 , nr_account =0 ;if (unlikely (nr_pages <=0 ))return 0 ;
@@ -5154,15 +5155,7 @@ unsigned long __alloc_pages_bulk(gfp_t g goto failed_irq;break ; } - -/* -* Ideally this would be batched but the best way to do -* that cheaply is to first convert zone_statistics to -* be inaccurate per-cpu counter like vm_events to avoid -* a RMW cycle then do the accounting with IRQs enabled. -*/ -__count_zid_vm_events (PGALLOC ,zone_idx (zone),1 ); -zone_statistics (ac.preferred_zoneref->zone, zone); + nr_account++;prep_new_page (page,0 , gfp,0 );if (page_list)
@@ -5172,6 +5165,9 @@ unsigned long __alloc_pages_bulk(gfp_t g nr_populated++; } +__count_zid_vm_events (PGALLOC ,zone_idx (zone), nr_account); +zone_statistics (ac.preferred_zoneref->zone, zone, nr_account); +local_unlock_irqrestore (&pagesets.lock, flags);return nr_populated;
_
Andrew Morton
maintainerPATCH 165/192 mm/page_alloc: reduce duration that IRQs are disabled for VM counters
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: reduce duration that IRQs are disabled for VM counters
IRQs are left disabled for the zone and node VM event counters. This is unnecessary as the affected counters are allowed to race for preemmption and IRQs.
This patch reduces the scope of IRQs being disabled via local_[lock|unlock]_irq on !PREEMPT_RT kernels. One __mod_zone_freepage_state is still called with IRQs disabled. While this could be moved out, it's not free on all architectures as some require IRQs to be disabled for mod_zone_page_state on !PREEMPT_RT kernels.
---
mm/page_alloc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-reduce-duration-that-irqs-are-disabled-for-vm-counters +6 -6
@@ -3530,11 +3530,11 @@ static struct page *rmqueue_pcplist(stru pcp =this_cpu_ptr (zone->per_cpu_pageset); list = &pcp->lists[migratetype]; page =__rmqueue_pcplist (zone, migratetype, alloc_flags, pcp, list); +local_unlock_irqrestore (&pagesets.lock, flags);if (page) {__count_zid_vm_events (PGALLOC ,page_zonenum (page),1 );zone_statistics (preferred_zone, zone,1 ); } -local_unlock_irqrestore (&pagesets.lock, flags);return page; }
@@ -3586,15 +3586,15 @@ struct page *rmqueue(struct zone *preferif (!page) page =__rmqueue (zone, order, migratetype, alloc_flags); }while (page &&check_new_pages (page, order)); -spin_unlock (&zone->lock);if (!page) goto failed; +__mod_zone_freepage_state (zone, -(1 << order),get_pcppage_migratetype (page)); +spin_unlock_irqrestore (&zone->lock, flags);__count_zid_vm_events (PGALLOC ,page_zonenum (page),1 << order);zone_statistics (preferred_zone, zone,1 ); -local_irq_restore (flags); out:/* Separate test+clear to avoid unnecessary atomics */
@@ -3607,7 +3607,7 @@ out:return page; failed: -local_irq_restore (flags); +spin_unlock_irqrestore (&zone->lock, flags);return NULL ; }
@@ -5165,11 +5165,11 @@ unsigned long __alloc_pages_bulk(gfp_t g nr_populated++; } +local_unlock_irqrestore (&pagesets.lock, flags); +__count_zid_vm_events (PGALLOC ,zone_idx (zone), nr_account);zone_statistics (ac.preferred_zoneref->zone, zone, nr_account); -local_unlock_irqrestore (&pagesets.lock, flags); -return nr_populated; failed_irq:
_
Andrew Morton
maintainerPATCH 166/192 mm/page_alloc: explicitly acquire the zone lock in __free_pages_ok
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: explicitly acquire the zone lock in __free_pages_ok
__free_pages_ok() disables IRQs before calling a common helper free_one_page() that acquires the zone lock. This is not safe according to Documentation/locking/locktypes.rst and in this context, IRQ disabling is not protecting a per_cpu_pages structure either or a local_lock would be used.
This patch explicitly acquires the lock with spin_lock_irqsave instead of relying on a helper. This removes the last instance of local_irq_save() in page_alloc.c.
---
mm/page_alloc.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-explicitly-acquire-the-zone-lock-in-__free_pages_ok +8 -8
@@ -1590,21 +1590,21 @@ static void __free_pages_ok(struct pageunsigned long flags;int migratetype;unsigned long pfn =page_to_pfn (page); +struct zone *zone =page_zone (page);if (!free_pages_prepare (page, order, true, fpi_flags))return ; migratetype =get_pfnblock_migratetype (page, pfn); -/* -* TODO FIX: Disable IRQs before acquiring IRQ-safe zone->lock -* and protect vmstat updates. -*/ -local_irq_save (flags); +spin_lock_irqsave (&zone->lock, flags);__count_vm_events (PGFREE ,1 << order); -free_one_page (page_zone (page), page, pfn, order, migratetype, - fpi_flags); -local_irq_restore (flags); +if (unlikely (has_isolate_pageblock (zone) || +is_migrate_isolate (migratetype))) { + migratetype =get_pfnblock_migratetype (page, pfn); + } +__free_one_page (page, pfn, zone, order, migratetype, fpi_flags); +spin_unlock_irqrestore (&zone->lock, flags); }void __free_pages_core (struct page *page,unsigned int order)
_
Andrew Morton
maintainerPATCH 167/192 mm/page_alloc: avoid conflating IRQs disabled with zone->lock
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: avoid conflating IRQs disabled with zone->lock
Historically when freeing pages, free_one_page() assumed that callers had IRQs disabled and the zone->lock could be acquired with spin_lock(). This confuses the scope of what local_lock_irq is protecting and what zone->lock is protecting in free_unref_page_list in particular.
This patch uses spin_lock_irqsave() for the zone->lock in free_one_page() instead of relying on callers to have disabled IRQs. free_unref_page_commit() is changed to only deal with PCP pages protected by the local lock. free_unref_page_list() then first frees isolated pages to the buddy lists with free_one_page() and frees the rest of the pages to the PCP via free_unref_page_commit(). The end result is that free_one_page() is no longer depending on side-effects of local_lock to be correct.
Note that this may incur a performance penalty while memory hot-remove is running but that is not a common operation.
---
mm/page_alloc.c | 75 ++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 26 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-avoid-conflating-irqs-disabled-with-zone-lock +49 -26
@@ -1501,13 +1501,15 @@ static void free_one_page(struct zone *zunsigned int order, int migratetype, fpi_t fpi_flags) { - spin_lock(&zone->lock); + unsigned long flags; + +spin_lock_irqsave (&zone->lock, flags);if (unlikely (has_isolate_pageblock (zone) ||is_migrate_isolate (migratetype))) { migratetype =get_pfnblock_migratetype (page, pfn); }__free_one_page (page, pfn, zone, order, migratetype, fpi_flags); -spin_unlock (&zone->lock); +spin_unlock_irqrestore (&zone->lock, flags); } static void __meminit __init_single_page(struct page *page, unsigned long pfn,
@@ -3285,31 +3287,13 @@ static bool free_unref_page_prepare(strureturn true; } -static void free_unref_page_commit (struct page *page,unsigned long pfn) +static void free_unref_page_commit (struct page *page,unsigned long pfn, +int migratetype) {struct zone *zone =page_zone (page);struct per_cpu_pages *pcp; -int migratetype; - migratetype =get_pcppage_migratetype (page);__count_vm_event (PGFREE ); - -/* -* We only track unmovable, reclaimable and movable on pcp lists. -* Free ISOLATE pages back to the allocator because they are being -* offlined but treat HIGHATOMIC as movable pages so we can get those -* areas back if necessary. Otherwise, we may have to free -* excessively into the page allocator -*/ -if (migratetype >=MIGRATE_PCPTYPES ) { -if (unlikely (is_migrate_isolate (migratetype))) { -free_one_page (zone, page, pfn,0 , migratetype, -FPI_NONE ); -return ; - } - migratetype =MIGRATE_MOVABLE ; - } - pcp =this_cpu_ptr (zone->per_cpu_pageset);list_add (&page->lru, &pcp->lists[migratetype]); pcp->count++;
@@ -3324,12 +3308,29 @@ void free_unref_page(struct page *page) {unsigned long flags;unsigned long pfn =page_to_pfn (page); +int migratetype;if (!free_unref_page_prepare (page, pfn))return ; +/* +* We only track unmovable, reclaimable and movable on pcp lists. +* Place ISOLATE pages on the isolated list because they are being +* offlined but treat HIGHATOMIC as movable pages so we can get those +* areas back if necessary. Otherwise, we may have to free +* excessively into the page allocator +*/ + migratetype =get_pcppage_migratetype (page); +if (unlikely (migratetype >=MIGRATE_PCPTYPES )) { +if (unlikely (is_migrate_isolate (migratetype))) { +free_one_page (page_zone (page), page, pfn,0 , migratetype,FPI_NONE ); +return ; + } + migratetype =MIGRATE_MOVABLE ; + } +local_lock_irqsave (&pagesets.lock, flags); -free_unref_page_commit (page, pfn); +free_unref_page_commit (page, pfn, migratetype);local_unlock_irqrestore (&pagesets.lock, flags); }
@@ -3341,22 +3342,44 @@ void free_unref_page_list(struct list_hestruct page *page, *next;unsigned long flags, pfn;int batch_count =0 ; +int migratetype;/* Prepare pages for freeing */ list_for_each_entry_safe (page, next, list, lru) { pfn =page_to_pfn (page);if (!free_unref_page_prepare (page, pfn))list_del (&page->lru); + +/* +* Free isolated pages directly to the allocator, see +* comment in free_unref_page. +*/ + migratetype =get_pcppage_migratetype (page); +if (unlikely (migratetype >=MIGRATE_PCPTYPES )) { +if (unlikely (is_migrate_isolate (migratetype))) { +list_del (&page->lru); +free_one_page (page_zone (page), page, pfn,0 , + migratetype,FPI_NONE ); +continue ; + } + +/* +* Non-isolated types over MIGRATE_PCPTYPES get added +* to the MIGRATE_MOVABLE pcp list. +*/ +set_pcppage_migratetype (page,MIGRATE_MOVABLE ); + } +set_page_private (page, pfn); }local_lock_irqsave (&pagesets.lock, flags);list_for_each_entry_safe (page, next, list, lru) { -unsigned long pfn =page_private (page); - + pfn =page_private (page);set_page_private (page,0 ); + migratetype =get_pcppage_migratetype (page);trace_mm_page_free_batched (page); -free_unref_page_commit (page, pfn); +free_unref_page_commit (page, pfn, migratetype);/* * Guard against excessive IRQ disabled times when we get
_
Andrew Morton
maintainerPATCH 168/192 mm/page_alloc: update PGFREE outside the zone lock in __free_pages_ok
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: update PGFREE outside the zone lock in __free_pages_ok
VM events do not need explicit protection by disabling IRQs so update the counter with IRQs enabled in __free_pages_ok.
---
mm/page_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-update-pgfree-outside-the-zone-lock-in-__free_pages_ok +2 -1
@@ -1600,13 +1600,14 @@ static void __free_pages_ok(struct page migratetype =get_pfnblock_migratetype (page, pfn);spin_lock_irqsave (&zone->lock, flags); -__count_vm_events (PGFREE ,1 << order);if (unlikely (has_isolate_pageblock (zone) ||is_migrate_isolate (migratetype))) { migratetype =get_pfnblock_migratetype (page, pfn); }__free_one_page (page, pfn, zone, order, migratetype, fpi_flags);spin_unlock_irqrestore (&zone->lock, flags); + +__count_vm_events (PGFREE ,1 << order); }void __free_pages_core (struct page *page,unsigned int order)
_
Andrew Morton
maintainerPATCH 169/192 mm: page_alloc: dump migrate-failed pages only at -EBUSY
In reply to Andrew MortonFrom: Minchan Kim <[email protected]>
Subject: mm: page_alloc: dump migrate-failed pages only at -EBUSY
alloc_contig_dump_pages() aims for helping debugging page migration failure by elevated page refcount compared to expected_count. (for the detail, please look at migrate_page_move_mapping)
However, -ENOMEM is just the case that system is under memory pressure state, not relevant with page refcount at all. Thus, the dumping page list is not helpful for the debugging point of view.
---
mm/page_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-dump-migrate-failed-pages-only-at-ebusy +2 -1
@@ -8800,7 +8800,8 @@ static int __alloc_contig_migrate_range(lru_cache_enable ();if (ret <0 ) { -alloc_contig_dump_pages (&cc->migratepages); +if (ret == -EBUSY ) +alloc_contig_dump_pages (&cc->migratepages);putback_movable_pages (&cc->migratepages);return ret; }
_
Andrew Morton
maintainerPATCH 170/192 mm/page_alloc: delete vm.percpu_pagelist_fraction
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: delete vm.percpu_pagelist_fraction
Patch series "Calculate pcp->high based on zone sizes and active CPUs", v2.
The per-cpu page allocator (PCP) is meant to reduce contention on the zone lock but the sizing of batch and high is archaic and neither takes the zone size into account or the number of CPUs local to a zone. With larger zones and more CPUs per node, the contention is getting worse. Furthermore, the fact that vm.percpu_pagelist_fraction adjusts both batch and high values means that the sysctl can reduce zone lock contention but also increase allocation latencies.
This series disassociates pcp->high from pcp->batch and then scales pcp->high based on the size of the local zone with limited impact to reclaim and accounting for active CPUs but leaves pcp->batch static. It also adapts the number of pages that can be on the pcp list based on recent freeing patterns.
The motivation is partially to adjust to larger memory sizes but is also driven by the fact that large batches of page freeing via release_pages() often shows zone contention as a major part of the problem. Another is a bug report based on an older kernel where a multi-terabyte process can takes several minutes to exit. A workaround was to use vm.percpu_pagelist_fraction to increase the pcp->high value but testing indicated that a production workload could not use the same values because of an increase in allocation latencies. Unfortunately, I cannot reproduce this test case myself as the multi-terabyte machines are in active use but it should alleviate the problem.
The series aims to address both and partially acts as a pre-requisite. pcp only works with order-0 which is useless for SLUB (when using high orders) and THP (unconditionally). To store high-order pages on PCP, the pcp->high values need to be increased first.
This patch (of 6):
The vm.percpu_pagelist_fraction is used to increase the batch and high limits for the per-cpu page allocator (PCP). The intent behind the sysctl is to reduce zone lock acquisition when allocating/freeing pages but it has a problem. While it can decrease contention, it can also increase latency on the allocation side due to unreasonably large batch sizes. This leads to games where an administrator adjusts percpu_pagelist_fraction on the fly to work around contention and allocation latency problems.
This series aims to alleviate the problems with zone lock contention while avoiding the allocation-side latency problems. For the purposes of review, it's easier to remove this sysctl now and reintroduce a similar sysctl later in the series that deals only with pcp->high.
---
Documentation/admin-guide/sysctl/vm.rst | 19 ------- include/linux/mmzone.h | 3 - kernel/sysctl.c | 8 --- mm/page_alloc.c | 55 +--------------------- 4 files changed, 4 insertions(+), 81 deletions(-)
Documentation/admin-guide/sysctl/vm.rst renamed from Documentation/admin-guide/sysctl/vm.rst~mm-page_alloc-delete-vmpercpu_pagelist_fraction +0 -19
@@ -64,7 +64,6 @@ Currently, these files are in /proc/sys/ - overcommit_ratio - page-cluster - panic_on_oom -- percpu_pagelist_fraction - stat_interval - stat_refresh - numa_stat
@@ -790,24 +789,6 @@ panic_on_oom=2+kdump gives you very stro why oom happens. You can get snapshot. -percpu_pagelist_fraction -======================== - -This is the fraction of pages at most (high mark pcp->high) in each zone that -are allocated for each per cpu page list. The min value for this is 8. It -means that we don't allow more than 1/8th of pages in each zone to be -allocated in any single per_cpu_pagelist. This entry only changes the value -of hot per cpu pagelists. User can specify a number like 100 to allocate -1/100th of each zone to each per cpu page list. - -The batch value of each per cpu pagelist is also updated as a result. It is -set to pcp->high/4. The upper limit of batch is (PAGE_SHIFT * 8) - -The initial value is zero. Kernel does not use this value at boot time to set -the high water marks for each per cpu page list. If the user writes '0' to this -sysctl, it will revert to this default behavior. - - stat_interval =============
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-delete-vmpercpu_pagelist_fraction +0 -3
@@ -1027,15 +1027,12 @@ int watermark_scale_factor_sysctl_handleextern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES ];int lowmem_reserve_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); -int percpu_pagelist_fraction_sysctl_handler (struct ctl_table *,int , -void *,size_t *,loff_t *);int sysctl_min_unmapped_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int sysctl_min_slab_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int numa_zonelist_order_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); -extern int percpu_pagelist_fraction;extern char numa_zonelist_order[];#define NUMA_ZONELIST_ORDER_LEN 16
kernel/sysctl.c renamed from kernel/sysctl.c~mm-page_alloc-delete-vmpercpu_pagelist_fraction +0 -8
@@ -2909,14 +2909,6 @@ static struct ctl_table vm_table[] = { .extra2 = &one_thousand, }, { - .procname = "percpu_pagelist_fraction", - .data = &percpu_pagelist_fraction, - .maxlen = sizeof(percpu_pagelist_fraction), - .mode = 0644, - .proc_handler = percpu_pagelist_fraction_sysctl_handler, - .extra1 = SYSCTL_ZERO, - }, - { .procname = "page_lock_unfairness", .data = &sysctl_page_lock_unfairness, .maxlen = sizeof(sysctl_page_lock_unfairness),
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-delete-vmpercpu_pagelist_fraction +4 -51
84 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 171/192 mm/page_alloc: disassociate the pcp->high from pcp->batch
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: disassociate the pcp->high from pcp->batch
The pcp high watermark is based on the batch size but there is no relationship between them other than it is convenient to use early in boot.
This patch takes the first step and bases pcp->high on the zone low watermark split across the number of CPUs local to a zone while the batch size remains the same to avoid increasing allocation latencies. The intent behind the default pcp->high is "set the number of PCP pages such that if they are all full that background reclaim is not started prematurely".
Note that in this patch the pcp->high values are adjusted after memory hotplug events, min_free_kbytes adjustments and watermark scale factor adjustments but not CPU hotplug events which is handled later in the series.
On a test KVM instance;
Before grep -E "high:|batch" /proc/zoneinfo | tail -2
high: 378
batch: 63After grep -E "high:|batch" /proc/zoneinfo | tail -2
high: 649
batch: 63[[email protected]: fix __setup_per_zone_wmarks for parallel memory hotplug]
---
mm/memory_hotplug.c | 6 ++-- mm/page_alloc.c | 62 +++++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 21 deletions(-)
mm/memory_hotplug.c renamed from mm/memory_hotplug.c~mm-page_alloc-disassociate-the-pcp-high-from-pcp-batch +3 -3
@@ -961,7 +961,6 @@ int __ref online_pages(unsigned long pfnnode_states_set_node (nid, &arg);if (need_zonelists_rebuild)build_all_zonelists (NULL ); -zone_pcp_update (zone);/* Basic onlining is complete, allow allocation of onlined pages. */ undo_isolate_page_range (pfn, pfn + nr_pages,MIGRATE_MOVABLE );
@@ -974,6 +973,7 @@ int __ref online_pages(unsigned long pfn*/ shuffle_zone (zone); +/* reinitialise watermarks and update pcp limits */ init_per_zone_wmark_min ();kswapd_run (nid);
@@ -1829,13 +1829,13 @@ int __ref offline_pages(unsigned long stadjust_managed_page_count (pfn_to_page (start_pfn), -nr_pages);adjust_present_page_count (zone, -nr_pages); +/* reinitialise watermarks and update pcp limits */ init_per_zone_wmark_min ();if (!populated_zone (zone)) {zone_pcp_reset (zone);build_all_zonelists (NULL ); - }else -zone_pcp_update (zone); + }node_states_clear_node (node, &arg);if (arg.status_change_nid >=0 ) {
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-disassociate-the-pcp-high-from-pcp-batch +44 -18
99 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 172/192 mm/page_alloc: adjust pcp->high after CPU hotplug events
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: adjust pcp->high after CPU hotplug events
The PCP high watermark is based on the number of online CPUs so the watermarks must be adjusted during CPU hotplug. At the time of hot-remove, the number of online CPUs is already adjusted but during hot-add, a delta needs to be applied to update PCP to the correct value. After this patch is applied, the high watermarks are adjusted correctly.
# grep high: /proc/zoneinfo | tail -1
high: 649
# echo 0 > /sys/devices/system/cpu/cpu4/online
# grep high: /proc/zoneinfo | tail -1
high: 664
# echo 1 > /sys/devices/system/cpu/cpu4/online
# grep high: /proc/zoneinfo | tail -1
high: 649---
include/linux/cpuhotplug.h | 2 - mm/internal.h | 2 - mm/page_alloc.c | 38 ++++++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 13 deletions(-)
include/linux/cpuhotplug.h renamed from include/linux/cpuhotplug.h~mm-page_alloc-adjust-pcp-high-after-cpu-hotplug-events +1 -1
@@ -54,7 +54,7 @@ enum cpuhp_state {CPUHP_MM_MEMCQ_DEAD ,CPUHP_PERCPU_CNT_DEAD ,CPUHP_RADIX_DEAD , -CPUHP_PAGE_ALLOC_DEAD , +CPUHP_PAGE_ALLOC ,CPUHP_NET_DEV_DEAD ,CPUHP_PCI_XGENE_DEAD , CPUHP_IOMMU_IOVA_DEAD,
mm/internal.h renamed from mm/internal.h~mm-page_alloc-adjust-pcp-high-after-cpu-hotplug-events +1 -1
@@ -206,7 +206,7 @@ extern int user_min_free_kbytes;extern void free_unref_page (struct page *page);extern void free_unref_page_list (struct list_head *list); -extern void zone_pcp_update (struct zone *zone); +extern void zone_pcp_update (struct zone *zone,int cpu_online);extern void zone_pcp_reset (struct zone *zone);extern void zone_pcp_disable (struct zone *zone);extern void zone_pcp_enable (struct zone *zone);
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-adjust-pcp-high-after-cpu-hotplug-events +27 -11
@@ -6667,7 +6667,7 @@ static int zone_batchsize(struct zone *z#endif } -static int zone_highsize (struct zone *zone,int batch) +static int zone_highsize (struct zone *zone,int batch,int cpu_online) {#ifdef CONFIG_MMU int high;
@@ -6678,9 +6678,10 @@ static int zone_highsize(struct zone *zo* so that if they are full then background reclaim will not be * started prematurely. The value is split across all online CPUs * local to the zone. Note that early in boot that CPUs may not be -* online yet. +* online yet and that during CPU hotplug that the cpumask is not +* yet updated when a CPU is being onlined. */ - nr_local_cpus =max (1U ,cpumask_weight (cpumask_of_node (zone_to_nid (zone)))); + nr_local_cpus =max (1U ,cpumask_weight (cpumask_of_node (zone_to_nid (zone)))) + cpu_online; high =low_wmark_pages (zone) / nr_local_cpus;/*
@@ -6754,12 +6755,12 @@ static void __zone_set_pageset_high_and_* Calculate and set new high and batch values for all per-cpu pagesets of a * zone based on the zone's size. */ -static void zone_set_pageset_high_and_batch (struct zone *zone) +static void zone_set_pageset_high_and_batch (struct zone *zone,int cpu_online) {int new_high, new_batch; new_batch =max (1 ,zone_batchsize (zone)); - new_high =zone_highsize (zone, new_batch); + new_high =zone_highsize (zone, new_batch, cpu_online);if (zone->pageset_high == new_high && zone->pageset_batch == new_batch)
@@ -6789,7 +6790,7 @@ void __meminit setup_zone_pageset(structper_cpu_pages_init (pcp, pzstats); } -zone_set_pageset_high_and_batch (zone); +zone_set_pageset_high_and_batch (zone,0 ); }/*
@@ -8044,6 +8045,7 @@ void __init set_dma_reserve(unsigned lonstatic int page_alloc_cpu_dead (unsigned int cpu) { +struct zone *zone;lru_add_drain_cpu (cpu);drain_pages (cpu);
@@ -8064,6 +8066,19 @@ static int page_alloc_cpu_dead(unsigned* race with what we are doing. */ cpu_vm_stats_fold (cpu); + +for_each_populated_zone (zone) +zone_pcp_update (zone,0 ); + +return 0 ; +} + +static int page_alloc_cpu_online (unsigned int cpu) +{ +struct zone *zone; + +for_each_populated_zone (zone) +zone_pcp_update (zone,1 );return 0 ; }
@@ -8089,8 +8104,9 @@ void __init page_alloc_init(void) hashdist =0 ;#endif - ret =cpuhp_setup_state_nocalls (CPUHP_PAGE_ALLOC_DEAD , -"mm/page_alloc:dead" ,NULL , + ret =cpuhp_setup_state_nocalls (CPUHP_PAGE_ALLOC , +"mm/page_alloc:pcp" , + page_alloc_cpu_online, page_alloc_cpu_dead);WARN_ON (ret <0 ); }
@@ -8252,7 +8268,7 @@ void setup_per_zone_wmarks(void)* and high limits or the limits may be inappropriate. */ for_each_zone (zone) -zone_pcp_update (zone); +zone_pcp_update (zone,0 ); }/*
@@ -9053,10 +9069,10 @@ EXPORT_SYMBOL(free_contig_range);* The zone indicated has a new number of managed_pages; batch sizes and percpu * page high values need to be recalculated. */ -void __meminit zone_pcp_update (struct zone *zone) +void zone_pcp_update (struct zone *zone,int cpu_online) {mutex_lock (&pcp_batch_high_lock); -zone_set_pageset_high_and_batch (zone); +zone_set_pageset_high_and_batch (zone, cpu_online);mutex_unlock (&pcp_batch_high_lock); }
_
Andrew Morton
maintainerPATCH 173/192 mm/page_alloc: scale the number of pages that are batch freed
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: scale the number of pages that are batch freed
When a task is freeing a large number of order-0 pages, it may acquire the zone->lock multiple times freeing pages in batches. This may unnecessarily contend on the zone lock when freeing very large number of pages. This patch adapts the size of the batch based on the recent pattern to scale the batch size for subsequent frees.
As the machines I used were not large enough to test this are not large enough to illustrate a problem, a debugging patch shows patterns like the following (slightly editted for clarity)
Baseline vanilla kernel
time-unmap-14426 [...] free_pcppages_bulk: free 63 count 378 high 378 time-unmap-14426 [...] free_pcppages_bulk: free 63 count 378 high 378 time-unmap-14426 [...] free_pcppages_bulk: free 63 count 378 high 378 time-unmap-14426 [...] free_pcppages_bulk: free 63 count 378 high 378 time-unmap-14426 [...] free_pcppages_bulk: free 63 count 378 high 378
With patches
time-unmap-7724 [...] free_pcppages_bulk: free 126 count 814 high 814 time-unmap-7724 [...] free_pcppages_bulk: free 252 count 814 high 814 time-unmap-7724 [...] free_pcppages_bulk: free 504 count 814 high 814 time-unmap-7724 [...] free_pcppages_bulk: free 751 count 814 high 814 time-unmap-7724 [...] free_pcppages_bulk: free 751 count 814 high 814
---
include/linux/mmzone.h | 3 +- mm/page_alloc.c | 41 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-scale-the-number-of-pages-that-are-batch-freed +2 -1
@@ -343,8 +343,9 @@ struct per_cpu_pages {int count;/* number of pages in the list */ int high;/* high watermark, emptying needed */ int batch;/* chunk size for buddy add/remove */ +short free_factor;/* batch scaling factor during free */ #ifdef CONFIG_NUMA -int expire;/* When 0, remote pagesets are drained */ +short expire;/* When 0, remote pagesets are drained */ #endif /* Lists of pages, one per migrate type stored on the pcp-lists */
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-scale-the-number-of-pages-that-are-batch-freed +39 -2
70 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 174/192 mm/page_alloc: limit the number of pages on PCP lists when reclaim is active
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: limit the number of pages on PCP lists when reclaim is active
When kswapd is active then direct reclaim is potentially active. In either case, it is possible that a zone would be balanced if pages were not trapped on PCP lists. Instead of draining remote pages, simply limit the size of the PCP lists while kswapd is active.
---
include/linux/mmzone.h | 1 + mm/page_alloc.c | 19 ++++++++++++++++++- mm/vmscan.c | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-limit-the-number-of-pages-on-pcp-lists-when-reclaim-is-active +1 -0
@@ -647,6 +647,7 @@ enum zone_flags {ZONE_BOOSTED_WATERMARK ,/* zone recently boosted watermarks. * Cleared when kswapd is woken. */ + ZONE_RECLAIM_ACTIVE, /* kswapd may be scanning the zone. */ };static inline unsigned long zone_managed_pages (struct zone *zone)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-limit-the-number-of-pages-on-pcp-lists-when-reclaim-is-active +18 -1
@@ -3302,6 +3302,23 @@ static int nr_pcp_free(struct per_cpu_pareturn batch; } +static int nr_pcp_high (struct per_cpu_pages *pcp,struct zone *zone) +{ +int high =READ_ONCE (pcp->high); + +if (unlikely (!high)) +return 0 ; + +if (!test_bit (ZONE_RECLAIM_ACTIVE , &zone->flags)) +return high; + +/* +* If reclaim is active, limit the number of pages that can be +* stored on pcp lists +*/ +return min (READ_ONCE (pcp->batch) <<2 , high); +} +static void free_unref_page_commit (struct page *page,unsigned long pfn,int migratetype) {
@@ -3313,7 +3330,7 @@ static void free_unref_page_commit(struc pcp =this_cpu_ptr (zone->per_cpu_pageset);list_add (&page->lru, &pcp->lists[migratetype]); pcp->count++; - high =READ_ONCE (pcp->high); + high =nr_pcp_high (pcp, zone);if (pcp->count >= high) {int batch =READ_ONCE (pcp->batch);
mm/vmscan.c renamed from mm/vmscan.c~mm-page_alloc-limit-the-number-of-pages-on-pcp-lists-when-reclaim-is-active +35 -0
@@ -3722,6 +3722,38 @@ static bool kswapd_shrink_node(pg_data_treturn sc->nr_scanned >= sc->nr_to_reclaim; } +/* Page allocator PCP high watermark is lowered if reclaim is active. */ +static inline void +update_reclaim_active (pg_data_t *pgdat,int highest_zoneidx,bool active) +{ +int i; +struct zone *zone; + +for (i =0 ; i <= highest_zoneidx; i++) { + zone = pgdat->node_zones + i; + +if (!managed_zone (zone)) +continue ; + +if (active) +set_bit (ZONE_RECLAIM_ACTIVE , &zone->flags); +else +clear_bit (ZONE_RECLAIM_ACTIVE , &zone->flags); + } +} + +static inline void +set_reclaim_active (pg_data_t *pgdat,int highest_zoneidx) +{ +update_reclaim_active (pgdat, highest_zoneidx, true); +} + +static inline void +clear_reclaim_active (pg_data_t *pgdat,int highest_zoneidx) +{ +update_reclaim_active (pgdat, highest_zoneidx, false); +} +/* * For kswapd, balance_pgdat() will reclaim pages across a node from zones * that are eligible for use by the caller until at least one zone is
@@ -3774,6 +3806,7 @@ static int balance_pgdat(pg_data_t *pgda boosted = nr_boost_reclaim; restart: +set_reclaim_active (pgdat, highest_zoneidx); sc.priority =DEF_PRIORITY ; do { unsigned long nr_reclaimed = sc.nr_reclaimed;
@@ -3907,6 +3940,8 @@ restart: pgdat->kswapd_failures++; out: +clear_reclaim_active (pgdat, highest_zoneidx); +/* If reclaim was boosted, account for the reclaim done in this pass */ if (boosted) {unsigned long flags;
_
Andrew Morton
maintainerPATCH 175/192 mm/page_alloc: introduce vm.percpu_pagelist_high_fraction
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: introduce vm.percpu_pagelist_high_fraction
This introduces a new sysctl vm.percpu_pagelist_high_fraction. It is similar to the old vm.percpu_pagelist_fraction. The old sysctl increased both pcp->batch and pcp->high with the higher pcp->high potentially reducing zone->lock contention. However, the higher pcp->batch value also potentially increased allocation latency while the PCP was refilled. This sysctl only adjusts pcp->high so that zone->lock contention is potentially reduced but allocation latency during a PCP refill remains the same.
# grep -E "high:|batch" /proc/zoneinfo | tail -2
high: 649
batch: 63 # sysctl vm.percpu_pagelist_high_fraction=8
# grep -E "high:|batch" /proc/zoneinfo | tail -2
high: 35071
batch: 63 # sysctl vm.percpu_pagelist_high_fraction=64
high: 4383
batch: 63 # sysctl vm.percpu_pagelist_high_fraction=0
high: 649
batch: 63---
Documentation/admin-guide/sysctl/vm.rst | 21 ++++++ include/linux/mmzone.h | 3 kernel/sysctl.c | 8 ++ mm/page_alloc.c | 69 +++++++++++++++++++--- 4 files changed, 94 insertions(+), 7 deletions(-)
Documentation/admin-guide/sysctl/vm.rst renamed from Documentation/admin-guide/sysctl/vm.rst~mm-page_alloc-introduce-vmpercpu_pagelist_high_fraction +21 -0
@@ -64,6 +64,7 @@ Currently, these files are in /proc/sys/ - overcommit_ratio - page-cluster - panic_on_oom +- percpu_pagelist_high_fraction - stat_interval - stat_refresh - numa_stat
@@ -789,6 +790,26 @@ panic_on_oom=2+kdump gives you very stro why oom happens. You can get snapshot. +percpu_pagelist_high_fraction +============================= + +This is the fraction of pages in each zone that are can be stored to +per-cpu page lists. It is an upper boundary that is divided depending +on the number of online CPUs. The min value for this is 8 which means +that we do not allow more than 1/8th of pages in each zone to be stored +on per-cpu page lists. This entry only changes the value of hot per-cpu +page lists. A user can specify a number like 100 to allocate 1/100th of +each zone between per-cpu lists. + +The batch value of each per-cpu page list remains the same regardless of +the value of the high fraction so allocation latencies are unaffected. + +The initial value is zero. Kernel uses this value to set the high pcp->high +mark based on the low watermark for the zone and the number of local +online CPUs. If the user writes '0' to this sysctl, it will revert to +this default behavior. + + stat_interval =============
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-introduce-vmpercpu_pagelist_high_fraction +3 -0
@@ -1029,12 +1029,15 @@ int watermark_scale_factor_sysctl_handleextern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES ];int lowmem_reserve_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); +int percpu_pagelist_high_fraction_sysctl_handler (struct ctl_table *,int , +void *,size_t *,loff_t *);int sysctl_min_unmapped_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int sysctl_min_slab_ratio_sysctl_handler (struct ctl_table *,int ,void *,size_t *,loff_t *);int numa_zonelist_order_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); +extern int percpu_pagelist_high_fraction;extern char numa_zonelist_order[];#define NUMA_ZONELIST_ORDER_LEN 16
kernel/sysctl.c renamed from kernel/sysctl.c~mm-page_alloc-introduce-vmpercpu_pagelist_high_fraction +8 -0
@@ -2909,6 +2909,14 @@ static struct ctl_table vm_table[] = { .extra2 = &one_thousand, }, { + .procname = "percpu_pagelist_high_fraction", + .data = &percpu_pagelist_high_fraction, + .maxlen = sizeof(percpu_pagelist_high_fraction), + .mode = 0644, + .proc_handler = percpu_pagelist_high_fraction_sysctl_handler, + .extra1 = SYSCTL_ZERO, + }, + { .procname = "page_lock_unfairness", .data = &sysctl_page_lock_unfairness, .maxlen = sizeof(sysctl_page_lock_unfairness),
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-introduce-vmpercpu_pagelist_high_fraction +62 -7
97 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 176/192 mm: drop SECTION_SHIFT in code comments
In reply to Andrew MortonFrom: Dong Aisheng <[email protected]>
Subject: mm: drop SECTION_SHIFT in code comments
Actually SECTIONS_SHIFT is used in the kernel code, so the code comments is strictly incorrect. And since commit bbeae5b05ef6 ("mm: move page flags layout to separate header"), SECTIONS_SHIFT definition has been moved to include/linux/page-flags-layout.h, since code itself looks quite straighforward, instead of moving the code comment into the new place as well, we just simply remove it.
This also fixed a checkpatch complain derived from the original code:
WARNING: please, no space before tabs
+ * SECTIONS_SHIFT ^I^I#bits space required to store a section #$
---
include/linux/mmzone.h | 2 -- 1 file changed, 2 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-drop-section_shift-in-code-comments +0 -2
@@ -1200,8 +1200,6 @@ static inline struct zoneref *first_zone#ifdef CONFIG_SPARSEMEM /* -* SECTION_SHIFT #bits space required to store a section # -* * PA_SECTION_SHIFT physical address to/from section number * PFN_SECTION_SHIFT pfn to/from section number */
_
Andrew Morton
maintainerPATCH 177/192 mm/page_alloc: improve memmap_pages dbg msg
In reply to Andrew MortonFrom: Dong Aisheng <[email protected]>
Subject: mm/page_alloc: improve memmap_pages dbg msg
Make debug message more accurate.
---
mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-improve-memmap_pages-dbg-msg +1 -1
@@ -7383,7 +7383,7 @@ static void __init free_area_init_core(spr_debug (" %s zone: %lu pages used for memmap\n" , zone_names[j], memmap_pages); }else -pr_warn (" %s zone: %lu pages exceeds freesize %lu\n" , +pr_warn (" %s zone: %lu memmap pages exceeds freesize %lu\n" , zone_names[j], memmap_pages, freesize); }
_
Andrew Morton
maintainerPATCH 178/192 mm/page_alloc: fix counting of managed_pages
In reply to Andrew MortonFrom: Liu Shixin <[email protected]>
Subject: mm/page_alloc: fix counting of managed_pages
commit f63661566fad ("mm/page_alloc.c: clear out zone->lowmem_reserve[] if the zone is empty") clears out zone->lowmem_reserve[] if zone is empty. But when zone is not empty and sysctl_lowmem_reserve_ratio[i] is set to zero, zone_managed_pages(zone) is not counted in the managed_pages either. This is inconsistent with the description of lowmem_reserve, so fix it.
---
mm/page_alloc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-fix-counting-of-managed_pages +6 -6
@@ -8240,14 +8240,14 @@ static void setup_per_zone_lowmem_reservunsigned long managed_pages =0 ;for (j = i +1 ; j <MAX_NR_ZONES ; j++) { -if (clear) { - zone->lowmem_reserve[j] =0 ; - }else { -struct zone *upper_zone = &pgdat->node_zones[j]; +struct zone *upper_zone = &pgdat->node_zones[j]; + + managed_pages +=zone_managed_pages (upper_zone); - managed_pages +=zone_managed_pages (upper_zone); +if (clear) + zone->lowmem_reserve[j] =0 ; +else zone->lowmem_reserve[j] = managed_pages / ratio; - } } } }
_
Andrew Morton
maintainerPATCH 179/192 mm/page_alloc: move free_the_page
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: move free_the_page
Patch series "Allow high order pages to be stored on PCP", v2.
The per-cpu page allocator (PCP) only handles order-0 pages. With the series "Use local_lock for pcp protection and reduce stat overhead" and "Calculate pcp->high based on zone sizes and active CPUs", it's now feasible to store high-order pages on PCP lists.
This small series allows PCP to store "cheap" orders where cheap is determined by PAGE_ALLOC_COSTLY_ORDER and THP-sized allocations.
This patch (of 2):
In the next page, free_compount_page is going to use the common helper free_the_page. This patch moves the definition to ease review. No functional change.
---
mm/page_alloc.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-move-free_the_page +8 -8
@@ -687,6 +687,14 @@ out:add_taint (TAINT_BAD_PAGE ,LOCKDEP_NOW_UNRELIABLE ); } +static inline void free_the_page (struct page *page,unsigned int order) +{ +if (order ==0 )/* Via pcp? */ +free_unref_page (page); +else +__free_pages_ok (page, order,FPI_NONE ); +} +/* * Higher-order pages are called "compound pages". They are structured thusly: *
@@ -5349,14 +5357,6 @@ unsigned long get_zeroed_page(gfp_t gfp_ }EXPORT_SYMBOL (get_zeroed_page); -static inline void free_the_page (struct page *page,unsigned int order) -{ -if (order ==0 )/* Via pcp? */ -free_unref_page (page); -else -__free_pages_ok (page, order,FPI_NONE ); -} -/** * __free_pages - Free pages allocated with alloc_pages(). * @page: The page pointer returned from alloc_pages().
_
Andrew Morton
maintainerPATCH 180/192 alpha: remove DISCONTIGMEM and NUMA
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: alpha: remove DISCONTIGMEM and NUMA
Patch series "Remove DISCONTIGMEM memory model", v3.
SPARSEMEM memory model was supposed to entirely replace DISCONTIGMEM a (long) while ago. The last architectures that used DISCONTIGMEM were updated to use other memory models in v5.11 and it is about the time to entirely remove DISCONTIGMEM from the kernel.
This set removes DISCONTIGMEM from alpha, arc and m68k, simplifies memory model selection in mm/Kconfig and replaces usage of redundant CONFIG_NEED_MULTIPLE_NODES and CONFIG_FLAT_NODE_MEM_MAP with CONFIG_NUMA and CONFIG_FLATMEM respectively.
I've also removed NUMA support on alpha that was BROKEN for more than 15 years.
There were also minor updates all over arch/ to remove mentions of DISCONTIGMEM in comments and #ifdefs.
This patch (of 9):
NUMA is marked broken on alpha for more than 15 years and DISCONTIGMEM was replaced with SPARSEMEM in v5.11.
Remove both NUMA and DISCONTIGMEM support from alpha.
---
arch/alpha/Kconfig | 22 -- arch/alpha/include/asm/machvec.h | 6 arch/alpha/include/asm/mmzone.h | 100 ------------ arch/alpha/include/asm/pgtable.h | 4 arch/alpha/include/asm/topology.h | 39 ---- arch/alpha/kernel/core_marvel.c | 53 ------ arch/alpha/kernel/core_wildfire.c | 29 --- arch/alpha/kernel/pci_iommu.c | 29 --- arch/alpha/kernel/proto.h | 8 - arch/alpha/kernel/setup.c | 16 -- arch/alpha/kernel/sys_marvel.c | 5 arch/alpha/kernel/sys_wildfire.c | 5 arch/alpha/mm/Makefile | 2 arch/alpha/mm/init.c | 3 arch/alpha/mm/numa.c | 223 ---------------------------- 15 files changed, 4 insertions(+), 540 deletions(-)
arch/alpha/include/asm/machvec.h renamed from arch/alpha/include/asm/machvec.h~alpha-remove-discontigmem-and-numa +0 -6
@@ -99,12 +99,6 @@ struct alpha_machine_vectorconst char *vector_name; -/* NUMA information */ -int (*pa_to_nid)(unsigned long ); -int (*cpuid_to_nid)(int ); -unsigned long (*node_mem_start)(int ); -unsigned long (*node_mem_size)(int ); -/* System specific parameters. */ union { struct {
arch/alpha/include/asm/mmzone.h deleted +0 -100
100 lines not on this page · raw message
arch/alpha/include/asm/pgtable.h renamed from arch/alpha/include/asm/pgtable.h~alpha-remove-discontigmem-and-numa +0 -4
@@ -206,7 +206,6 @@ extern unsigned long __zero_page(void);#define page_to_pa (page) (page_to_pfn(page) << PAGE_SHIFT)#define pte_pfn (pte) (pte_val(pte) >> 32) -#ifndef CONFIG_DISCONTIGMEM #define pte_page (pte) pfn_to_page(pte_pfn(pte))#define mk_pte (page, pgprot) \ ({ \
@@ -215,7 +214,6 @@ extern unsigned long __zero_page(void);pte_val (pte) = (page_to_pfn (page) <<32 ) |pgprot_val (pgprot); \ pte; \ }) -#endif extern inline pte_t pfn_pte (unsigned long physpfn,pgprot_t pgprot) {pte_t pte;pte_val (pte) = (PHYS_TWIDDLE (physpfn) <<32 ) |pgprot_val (pgprot);return pte; }
@@ -330,9 +328,7 @@ extern inline pte_t mk_swap_pte(unsigned#define __pte_to_swp_entry (pte) ((swp_entry_t) { pte_val(pte) })#define __swp_entry_to_pte (x) ((pte_t) { (x).val }) -#ifndef CONFIG_DISCONTIGMEM #define kern_addr_valid (addr) (1) -#endif #define pte_ERROR (e) \ printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
arch/alpha/include/asm/topology.h renamed from arch/alpha/include/asm/topology.h~alpha-remove-discontigmem-and-numa +0 -39
@@ -7,45 +7,6 @@#include <linux/numa.h> #include <asm/machvec.h> -#ifdef CONFIG_NUMA -static inline int cpu_to_node (int cpu) -{ -int node; - -if (!alpha_mv.cpuid_to_nid) -return 0 ; - - node = alpha_mv.cpuid_to_nid (cpu); - -#ifdef DEBUG_NUMA -BUG_ON (node <0 ); -#endif - -return node; -} - -extern struct cpumask node_to_cpumask_map[]; -/* FIXME: This is dumb, recalculating every time. But simple. */ -static const struct cpumask *cpumask_of_node (int node) -{ -int cpu; - -if (node ==NUMA_NO_NODE ) -return cpu_all_mask; - -cpumask_clear (&node_to_cpumask_map[node]); - -for_each_online_cpu (cpu) { -if (cpu_to_node (cpu) == node) -cpumask_set_cpu (cpu, node_to_cpumask_map[node]); - } - -return &node_to_cpumask_map[node]; -} - -#define cpumask_of_pcibus (bus) (cpu_online_mask) - -#endif /* !CONFIG_NUMA */ # include <asm-generic/topology.h> #endif /* _ASM_ALPHA_TOPOLOGY_H */
arch/alpha/Kconfig renamed from arch/alpha/Kconfig~alpha-remove-discontigmem-and-numa +0 -22
@@ -549,29 +549,12 @@ config NR_CPUS MARVEL support can handle a maximum of 32 CPUs, all the others with working support have a maximum of 4 CPUs. -config ARCH_DISCONTIGMEM_ENABLE - bool "Discontiguous Memory Support" - depends on BROKEN - help - Say Y to support efficient handling of discontiguous physical memory, - for architectures which are either NUMA (Non-Uniform Memory Access) - or have huge holes in the physical address space for other reasons. - See <file:Documentation/vm/numa.rst> for more. - config ARCH_SPARSEMEM_ENABLE bool "Sparse Memory Support" help Say Y to support efficient handling of discontiguous physical memory, for systems that have huge holes in the physical address space. -config NUMA - bool "NUMA Support (EXPERIMENTAL)" - depends on DISCONTIGMEM && BROKEN - help - Say Y to compile the kernel to support NUMA (Non-Uniform Memory - Access). This option is for configuring high-end multiprocessor - server machines. If in doubt, say N. - config ALPHA_WTINT bool "Use WTINT" if ALPHA_SRM || ALPHA_GENERIC default y if ALPHA_QEMU
@@ -596,11 +579,6 @@ config ALPHA_WTINT If unsure, say N. -config NODES_SHIFT - int - default "7" - depends on NEED_MULTIPLE_NODES - # LARGE_VMALLOC is racy, if you *really* need it then fix it first config ALPHA_LARGE_VMALLOC bool
arch/alpha/kernel/core_marvel.c renamed from arch/alpha/kernel/core_marvel.c~alpha-remove-discontigmem-and-numa +3 -50
72 lines not on this page · raw message
arch/alpha/kernel/core_wildfire.c renamed from arch/alpha/kernel/core_wildfire.c~alpha-remove-discontigmem-and-numa +1 -28
@@ -434,39 +434,12 @@ wildfire_write_config(struct pci_bus *bureturn PCIBIOS_SUCCESSFUL ; } -struct pci_ops wildfire_pci_ops = +struct pci_ops wildfire_pci_ops = { .read = wildfire_read_config, .write = wildfire_write_config, }; - -/* -* NUMA Support -*/ -int wildfire_pa_to_nid (unsigned long pa) -{ -return pa >>36 ; -} - -int wildfire_cpuid_to_nid (int cpuid) -{ -/* assume 4 CPUs per node */ -return cpuid >>2 ; -} - -unsigned long wildfire_node_mem_start (int nid) -{ -/* 64GB per node */ -return (unsigned long )nid * (64UL *1024 *1024 *1024 ); -} - -unsigned long wildfire_node_mem_size (int nid) -{ -/* 64GB per node */ -return 64UL *1024 *1024 *1024 ; -} -#if DEBUG_DUMP_REGS static void __init
arch/alpha/kernel/pci_iommu.c renamed from arch/alpha/kernel/pci_iommu.c~alpha-remove-discontigmem-and-numa +0 -29
@@ -71,33 +71,6 @@ iommu_arena_new_node(int nid, struct pciif (align < mem_size) align = mem_size; - -#ifdef CONFIG_DISCONTIGMEM - - arena =memblock_alloc_node (sizeof (*arena), align, nid); -if (!NODE_DATA (nid) || !arena) { -printk ("%s: couldn't allocate arena from node %d\n" -" falling back to system-wide allocation\n" , - __func__, nid); - arena =memblock_alloc (sizeof (*arena),SMP_CACHE_BYTES ); -if (!arena) -panic ("%s: Failed to allocate %zu bytes\n" , __func__, -sizeof (*arena)); - } - - arena->ptes =memblock_alloc_node (sizeof (*arena), align, nid); -if (!NODE_DATA (nid) || !arena->ptes) { -printk ("%s: couldn't allocate arena ptes from node %d\n" -" falling back to system-wide allocation\n" , - __func__, nid); - arena->ptes =memblock_alloc (mem_size, align); -if (!arena->ptes) -panic ("%s: Failed to allocate %lu bytes align=0x%lx\n" , - __func__, mem_size, align); - } - -#else /* CONFIG_DISCONTIGMEM */ - arena =memblock_alloc (sizeof (*arena),SMP_CACHE_BYTES );if (!arena) panic("%s: Failed to allocate %zu bytes\n", __func__,
@@ -107,8 +80,6 @@ iommu_arena_new_node(int nid, struct pcipanic ("%s: Failed to allocate %lu bytes align=0x%lx\n" , __func__, mem_size, align); -#endif /* CONFIG_DISCONTIGMEM */ -spin_lock_init (&arena->lock); arena->hose = hose; arena->dma_base = base;
arch/alpha/kernel/proto.h renamed from arch/alpha/kernel/proto.h~alpha-remove-discontigmem-and-numa +0 -8
@@ -49,10 +49,6 @@ extern void marvel_init_arch(void);extern void marvel_kill_arch (int );extern void marvel_machine_check (unsigned long ,unsigned long );extern void marvel_pci_tbi (struct pci_controller *,dma_addr_t ,dma_addr_t ); -extern int marvel_pa_to_nid (unsigned long ); -extern int marvel_cpuid_to_nid (int ); -extern unsigned long marvel_node_mem_start (int ); -extern unsigned long marvel_node_mem_size (int );extern struct _alpha_agp_info *marvel_agp_info (void );struct io7 *marvel_find_io7 (int pe);struct io7 *marvel_next_io7 (struct io7 *prev);
@@ -101,10 +97,6 @@ extern void wildfire_init_arch(void);extern void wildfire_kill_arch (int );extern void wildfire_machine_check (unsigned long vector,unsigned long la_ptr);extern void wildfire_pci_tbi (struct pci_controller *,dma_addr_t ,dma_addr_t ); -extern int wildfire_pa_to_nid (unsigned long ); -extern int wildfire_cpuid_to_nid (int ); -extern unsigned long wildfire_node_mem_start (int ); -extern unsigned long wildfire_node_mem_size (int );/* console.c */ #ifdef CONFIG_VGA_HOSE
arch/alpha/kernel/setup.c renamed from arch/alpha/kernel/setup.c~alpha-remove-discontigmem-and-numa +0 -16
@@ -79,11 +79,6 @@ int alpha_l3_cacheshape;unsigned long alpha_verbose_mcheck =CONFIG_VERBOSE_MCHECK_ON ;#endif -#ifdef CONFIG_NUMA -struct cpumask node_to_cpumask_map[MAX_NUMNODES ]__read_mostly ; -EXPORT_SYMBOL (node_to_cpumask_map); -#endif -/* Which processor we booted from. */ int boot_cpuid;
@@ -305,7 +300,6 @@ move_initrd(unsigned long mem_limit) }#endif -#ifndef CONFIG_DISCONTIGMEM static void __init setup_memory (void *kernel_end) {
@@ -389,9 +383,6 @@ setup_memory(void *kernel_end) }#endif /* CONFIG_BLK_DEV_INITRD */ } -#else -extern void setup_memory (void *); -#endif /* !CONFIG_DISCONTIGMEM */ int __init page_is_ram (unsigned long pfn)
@@ -618,13 +609,6 @@ setup_arch(char **cmdline_p) "VERBOSE_MCHECK "#endif -#ifdef CONFIG_DISCONTIGMEM -"DISCONTIGMEM " -#ifdef CONFIG_NUMA - "NUMA " -#endif -#endif -#ifdef CONFIG_DEBUG_SPINLOCK "DEBUG_SPINLOCK "#endif
arch/alpha/kernel/sys_marvel.c renamed from arch/alpha/kernel/sys_marvel.c~alpha-remove-discontigmem-and-numa +0 -5
@@ -461,10 +461,5 @@ struct alpha_machine_vector marvel_ev7_m .kill_arch = marvel_kill_arch, .pci_map_irq = marvel_map_irq, .pci_swizzle = common_swizzle, - - .pa_to_nid = marvel_pa_to_nid, - .cpuid_to_nid = marvel_cpuid_to_nid, - .node_mem_start = marvel_node_mem_start, - .node_mem_size = marvel_node_mem_size, };ALIAS_MV (marvel_ev7)
arch/alpha/kernel/sys_wildfire.c renamed from arch/alpha/kernel/sys_wildfire.c~alpha-remove-discontigmem-and-numa +0 -5
@@ -337,10 +337,5 @@ struct alpha_machine_vector wildfire_mv .kill_arch = wildfire_kill_arch, .pci_map_irq = wildfire_map_irq, .pci_swizzle = common_swizzle, - - .pa_to_nid = wildfire_pa_to_nid, - .cpuid_to_nid = wildfire_cpuid_to_nid, - .node_mem_start = wildfire_node_mem_start, - .node_mem_size = wildfire_node_mem_size, };ALIAS_MV (wildfire)
arch/alpha/mm/init.c renamed from arch/alpha/mm/init.c~alpha-remove-discontigmem-and-numa +0 -3
@@ -235,8 +235,6 @@ callback_init(void * kernel_end)return kernel_end; } - -#ifndef CONFIG_DISCONTIGMEM /* * paging_init() sets up the memory map. */
@@ -257,7 +255,6 @@ void __init paging_init(void)/* Initialize the kernel's ZERO_PGE. */ memset ((void *)ZERO_PGE ,0 ,PAGE_SIZE ); } -#endif /* CONFIG_DISCONTIGMEM */ #if defined(CONFIG_ALPHA_GENERIC ) || defined(CONFIG_ALPHA_SRM )void
arch/alpha/mm/Makefile renamed from arch/alpha/mm/Makefile~alpha-remove-discontigmem-and-numa +0 -2
@@ -6,5 +6,3 @@ ccflags-y := -Werror obj-y := init.o fault.o - -obj-$(CONFIG_DISCONTIGMEM) += numa.o
arch/alpha/mm/numa.c deleted +0 -223
223 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 181/192 arc: update comment about HIGHMEM implementation
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: arc: update comment about HIGHMEM implementation
Arc does not use DISCONTIGMEM to implement high memory, update the comment describing how high memory works to reflect this.
---
arch/arc/mm/init.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)
arch/arc/mm/init.c renamed from arch/arc/mm/init.c~arc-update-comment-about-highmem-implementation +5 -8
@@ -139,16 +139,13 @@ void __init setup_arch_memory(void)#ifdef CONFIG_HIGHMEM /* -* Populate a new node with highmem -* * On ARC (w/o PAE) HIGHMEM addresses are actually smaller (0 based) -* than addresses in normal ala low memory (0x8000_0000 based). +* than addresses in normal aka low memory (0x8000_0000 based). * Even with PAE, the huge peripheral space hole would waste a lot of -* mem with single mem_map[]. This warrants a mem_map per region design. -* Thus HIGHMEM on ARC is imlemented with DISCONTIGMEM. -* -* DISCONTIGMEM in turns requires multiple nodes. node 0 above is -* populated with normal memory zone while node 1 only has highmem +* mem with single contiguous mem_map[]. +* Thus when HIGHMEM on ARC is enabled the memory map corresponding +* to the hole is freed and ARC specific version of pfn_valid() +* handles the hole in the memory map. */ #ifdef CONFIG_DISCONTIGMEM node_set_online (1 );
_
Andrew Morton
maintainerPATCH 182/192 arc: remove support for DISCONTIGMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: arc: remove support for DISCONTIGMEM
DISCONTIGMEM was replaced by FLATMEM with freeing of the unused memory map in v5.11.
Remove the support for DISCONTIGMEM entirely.
---
arch/arc/Kconfig | 13 ---------- arch/arc/include/asm/mmzone.h | 40 -------------------------------- arch/arc/mm/init.c | 8 ------ 3 files changed, 61 deletions(-)
arch/arc/include/asm/mmzone.h deleted +0 -40
@@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* -* Copyright (C) 2016 Synopsys, Inc. (www.synopsys.com) -*/ - -#ifndef _ASM_ARC_MMZONE_H -#define _ASM_ARC_MMZONE_H - -#ifdef CONFIG_DISCONTIGMEM - -extern struct pglist_data node_data[]; -#define NODE_DATA (nid) (&node_data[nid]) - -static inline int pfn_to_nid (unsigned long pfn) -{ -int is_end_low =1 ; - -if (IS_ENABLED (CONFIG_ARC_HAS_PAE40 )) - is_end_low = pfn <=virt_to_pfn (0xFFFFFFFFUL ); - -/* -* node 0: lowmem: 0x8000_0000 to 0xFFFF_FFFF -* node 1: HIGHMEM w/o PAE40: 0x0 to 0x7FFF_FFFF -* HIGHMEM with PAE40: 0x1_0000_0000 to ... -*/ -if (pfn >=ARCH_PFN_OFFSET && is_end_low) -return 0 ; - -return 1 ; -} - -static inline int pfn_valid (unsigned long pfn) -{ -int nid =pfn_to_nid (pfn); - -return (pfn <=node_end_pfn (nid)); -} -#endif /* CONFIG_DISCONTIGMEM */ - -#endif
arch/arc/Kconfig renamed from arch/arc/Kconfig~arc-remove-support-for-discontigmem +0 -13
@@ -62,10 +62,6 @@ config SCHED_OMIT_FRAME_POINTER config GENERIC_CSUM def_bool y -config ARCH_DISCONTIGMEM_ENABLE - def_bool n - depends on BROKEN - config ARCH_FLATMEM_ENABLE def_bool y
@@ -344,15 +340,6 @@ config ARC_HUGEPAGE_16M endchoice -config NODES_SHIFT - int "Maximum NUMA Nodes (as a power of 2)" - default "0" if !DISCONTIGMEM - default "1" if DISCONTIGMEM - depends on NEED_MULTIPLE_NODES - help - Accessing memory beyond 1GB (with or w/o PAE) requires 2 memory - zones. - config ARC_COMPACT_IRQ_LEVELS depends on ISA_ARCOMPACT bool "Setup Timer IRQ as high Priority"
arch/arc/mm/init.c renamed from arch/arc/mm/init.c~arc-remove-support-for-discontigmem +0 -8
@@ -32,11 +32,6 @@ unsigned long arch_pfn_offset;EXPORT_SYMBOL (arch_pfn_offset);#endif -#ifdef CONFIG_DISCONTIGMEM -struct pglist_data node_data[MAX_NUMNODES ]__read_mostly ; -EXPORT_SYMBOL (node_data); -#endif -long __init arc_get_mem_sz (void ) {return low_mem_sz;
@@ -147,9 +142,6 @@ void __init setup_arch_memory(void)* to the hole is freed and ARC specific version of pfn_valid() * handles the hole in the memory map. */ -#ifdef CONFIG_DISCONTIGMEM -node_set_online (1 ); -#endif min_high_pfn =PFN_DOWN (high_mem_start); max_high_pfn =PFN_DOWN (high_mem_start + high_mem_sz);
_
Andrew Morton
maintainerPATCH 183/192 m68k: remove support for DISCONTIGMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: m68k: remove support for DISCONTIGMEM
DISCONTIGMEM was replaced by FLATMEM with freeing of the unused memory map in v5.11.
Remove the support for DISCONTIGMEM entirely.
---
arch/m68k/Kconfig.cpu | 10 -------- arch/m68k/include/asm/mmzone.h | 10 -------- arch/m68k/include/asm/page.h | 2 - arch/m68k/include/asm/page_mm.h | 35 ------------------------------ arch/m68k/mm/init.c | 20 ----------------- 5 files changed, 1 insertion(+), 76 deletions(-)
arch/m68k/include/asm/mmzone.h deleted +0 -10
@@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_M68K_MMZONE_H_ -#define _ASM_M68K_MMZONE_H_ - -extern pg_data_t pg_data_map[]; - -#define NODE_DATA (nid) (&pg_data_map[nid]) -#define NODE_MEM_MAP (nid) (NODE_DATA(nid)->node_mem_map) - -#endif /* _ASM_M68K_MMZONE_H_ */
arch/m68k/include/asm/page.h renamed from arch/m68k/include/asm/page.h~m68k-remove-support-for-discontigmem +1 -1
@@ -62,7 +62,7 @@ extern unsigned long _ramend;#include <asm/page_no.h> #endif -#if !defined(CONFIG_MMU ) || defined(CONFIG_DISCONTIGMEM ) +#ifndef CONFIG_MMU #define __phys_to_pfn (paddr) ((unsigned long)((paddr) >> PAGE_SHIFT))#define __pfn_to_phys (pfn) PFN_PHYS(pfn)#endif
arch/m68k/include/asm/page_mm.h renamed from arch/m68k/include/asm/page_mm.h~m68k-remove-support-for-discontigmem +0 -35
@@ -126,26 +126,6 @@ static inline void *__va(unsigned long xextern int m68k_virt_to_node_shift; -#ifndef CONFIG_DISCONTIGMEM -#define __virt_to_node (addr) (&pg_data_map[0]) -#else -extern struct pglist_data *pg_data_table[]; - -static inline __attribute_const__ int __virt_to_node_shift(void) -{ -int shift; - - asm ( - "1: moveq #0,%0\n" - m68k_fixup(%c1, 1b) - : "=d" (shift) - : "i" (m68k_fixup_vnode_shift)); -return shift; -} - -#define __virt_to_node (addr) (pg_data_table[(unsigned long)(addr) >> __virt_to_node_shift()]) -#endif -#define virt_to_page (addr) ({ \ pfn_to_page(virt_to_pfn(addr)); \ })
@@ -153,23 +133,8 @@ static inline __attribute_const__ int __pfn_to_virt (page_to_pfn (page)); \ }) -#ifdef CONFIG_DISCONTIGMEM -#define pfn_to_page (pfn) ({ \ - unsigned long __pfn = (pfn); \ - struct pglist_data *pgdat; \ - pgdat = __virt_to_node((unsigned long)pfn_to_virt(__pfn)); \ - pgdat->node_mem_map + (__pfn - pgdat->node_start_pfn); \ -}) -#define page_to_pfn (_page) ({ \ - const struct page *__p = (_page); \ - struct pglist_data *pgdat; \ - pgdat = &pg_data_map[page_to_nid(__p)]; \ - ((__p) - pgdat->node_mem_map) + pgdat->node_start_pfn; \ -}) -#else #define ARCH_PFN_OFFSET (m68k_memory[0].addr >> PAGE_SHIFT)#include <asm-generic/memory_model.h> -#endif #define virt_addr_valid (kaddr) ((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory)#define pfn_valid (pfn) virt_addr_valid(pfn_to_virt(pfn))
arch/m68k/Kconfig.cpu renamed from arch/m68k/Kconfig.cpu~m68k-remove-support-for-discontigmem +0 -10
@@ -408,10 +408,6 @@ config SINGLE_MEMORY_CHUNK order" to save memory that could be wasted for unused memory map. Say N if not sure. -config ARCH_DISCONTIGMEM_ENABLE - depends on BROKEN - def_bool MMU && !SINGLE_MEMORY_CHUNK - config FORCE_MAX_ZONEORDER int "Maximum zone order" if ADVANCED depends on !SINGLE_MEMORY_CHUNK
@@ -451,11 +447,6 @@ config M68K_L2_CACHE depends on MAC default y -config NODES_SHIFT - int - default "3" - depends on DISCONTIGMEM - config CPU_HAS_NO_BITFIELDS bool
@@ -553,4 +544,3 @@ config CACHE_COPYBACK The ColdFire CPU cache is set into Copy-back mode. endchoice endif -
arch/m68k/mm/init.c renamed from arch/m68k/mm/init.c~m68k-remove-support-for-discontigmem +0 -20
@@ -44,28 +44,8 @@ EXPORT_SYMBOL(empty_zero_page);int m68k_virt_to_node_shift; -#ifdef CONFIG_DISCONTIGMEM -pg_data_t pg_data_map[MAX_NUMNODES ]; -EXPORT_SYMBOL (pg_data_map); - -pg_data_t *pg_data_table[65 ]; -EXPORT_SYMBOL (pg_data_table); -#endif -void __init m68k_setup_node (int node) { -#ifdef CONFIG_DISCONTIGMEM -struct m68k_mem_info *info = m68k_memory + node; -int i, end; - - i = (unsigned long )phys_to_virt (info->addr) >>__virt_to_node_shift (); - end = (unsigned long )phys_to_virt (info->addr + info->size -1 ) >>__virt_to_node_shift (); -for (; i <= end; i++) { -if (pg_data_table[i]) -pr_warn ("overlap at %u for chunk %u\n" , i, node); - pg_data_table[i] = pg_data_map + node; - } -#endif node_set_online (node); }
_
Andrew Morton
maintainerPATCH 184/192 mm: remove CONFIG_DISCONTIGMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: mm: remove CONFIG_DISCONTIGMEM
There are no architectures that support DISCONTIGMEM left.
Remove the configuration option and the dead code it was guarding in the generic memory management code.
---
include/asm-generic/memory_model.h | 37 ++------------------------- include/linux/mmzone.h | 8 +++-- mm/Kconfig | 25 ++---------------- mm/page_alloc.c | 13 --------- 4 files changed, 12 insertions(+), 71 deletions(-)
include/asm-generic/memory_model.h renamed from include/asm-generic/memory_model.h~mm-remove-config_discontigmem +4 -33
@@ -6,47 +6,18 @@#ifndef __ASSEMBLY__ +/* +* supports 3 memory models. +*/ #if defined(CONFIG_FLATMEM )#ifndef ARCH_PFN_OFFSET #define ARCH_PFN_OFFSET (0UL)#endif -#elif defined(CONFIG_DISCONTIGMEM ) - -#ifndef arch_pfn_to_nid -#define arch_pfn_to_nid (pfn) pfn_to_nid(pfn) -#endif - -#ifndef arch_local_page_offset -#define arch_local_page_offset (pfn, nid) \ - ((pfn) - NODE_DATA(nid)->node_start_pfn) -#endif - -#endif /* CONFIG_DISCONTIGMEM */ - -/* -* supports 3 memory models. -*/ -#if defined(CONFIG_FLATMEM ) -#define __pfn_to_page (pfn) (mem_map + ((pfn) - ARCH_PFN_OFFSET))#define __page_to_pfn (page) ((unsigned long)((page) - mem_map) + \ ARCH_PFN_OFFSET) -#elif defined(CONFIG_DISCONTIGMEM ) - -#define __pfn_to_page (pfn) \ -({ unsigned long __pfn = (pfn); \ - unsigned long __nid = arch_pfn_to_nid(__pfn); \ - NODE_DATA(__nid)->node_mem_map + arch_local_page_offset(__pfn, __nid);\ -}) - -#define __page_to_pfn (pg) \ -({ const struct page *__pg = (pg); \ - struct pglist_data *__pgdat = NODE_DATA(page_to_nid(__pg)); \ - (unsigned long)(__pg - __pgdat->node_mem_map) + \ - __pgdat->node_start_pfn; \ -})#elif defined(CONFIG_SPARSEMEM_VMEMMAP )
@@ -70,7 +41,7 @@struct mem_section *__sec =__pfn_to_section (__pfn); \__section_mem_map_addr (__sec) + __pfn; \ }) -#endif /* CONFIG_FLATMEM/DISCONTIGMEM/SPARSEMEM */ +#endif /* CONFIG_FLATMEM/SPARSEMEM */ /* * Convert a physical address to a Page Frame Number and back
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-remove-config_discontigmem +5 -3
@@ -749,10 +749,12 @@ struct zonelist {struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST +1 ]; }; -#ifndef CONFIG_DISCONTIGMEM -/* The array of struct pages - for discontigmem use pgdat->lmem_map */ +/* +* The array of struct pages for flatmem. +* It must be declared for SPARSEMEM as well because there are configurations +* that rely on that. +*/ extern struct page *mem_map; -#endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE struct deferred_split {
mm/Kconfig renamed from mm/Kconfig~mm-remove-config_discontigmem +3 -22
@@ -19,7 +19,7 @@ choice config FLATMEM_MANUAL bool "Flat Memory" - depends on !(ARCH_DISCONTIGMEM_ENABLE || ARCH_SPARSEMEM_ENABLE) || ARCH_FLATMEM_ENABLE + depends on !ARCH_SPARSEMEM_ENABLE || ARCH_FLATMEM_ENABLE help This option is best suited for non-NUMA systems with flat address space. The FLATMEM is the most efficient
@@ -32,21 +32,6 @@ config FLATMEM_MANUAL If unsure, choose this option (Flat Memory) over any other. -config DISCONTIGMEM_MANUAL - bool "Discontiguous Memory" - depends on ARCH_DISCONTIGMEM_ENABLE - help - This option provides enhanced support for discontiguous - memory systems, over FLATMEM. These systems have holes - in their physical address spaces, and this option provides - more efficient handling of these holes. - - Although "Discontiguous Memory" is still used by several - architectures, it is considered deprecated in favor of - "Sparse Memory". - - If unsure, choose "Sparse Memory" over this option. - config SPARSEMEM_MANUAL bool "Sparse Memory" depends on ARCH_SPARSEMEM_ENABLE
@@ -62,17 +47,13 @@ config SPARSEMEM_MANUAL endchoice -config DISCONTIGMEM - def_bool y - depends on (!SELECT_MEMORY_MODEL && ARCH_DISCONTIGMEM_ENABLE) || DISCONTIGMEM_MANUAL - config SPARSEMEM def_bool y depends on (!SELECT_MEMORY_MODEL && ARCH_SPARSEMEM_ENABLE) || SPARSEMEM_MANUAL config FLATMEM def_bool y - depends on (!DISCONTIGMEM && !SPARSEMEM) || FLATMEM_MANUAL + depends on !SPARSEMEM || FLATMEM_MANUAL config FLAT_NODE_MEM_MAP def_bool y
@@ -85,7 +66,7 @@ config FLAT_NODE_MEM_MAP # config NEED_MULTIPLE_NODES def_bool y - depends on DISCONTIGMEM || NUMA + depends on NUMA # # SPARSEMEM_EXTREME (which is the default) does some bootmem
mm/page_alloc.c renamed from mm/page_alloc.c~mm-remove-config_discontigmem +0 -13
@@ -349,20 +349,7 @@ compound_page_dtor * const compound_pageint min_free_kbytes =1024 ;int user_min_free_kbytes =-1 ; -#ifdef CONFIG_DISCONTIGMEM -/* -* DiscontigMem defines memory ranges as separate pg_data_t even if the ranges -* are not on separate NUMA nodes. Functionally this works but with -* watermark_boost_factor, it can reclaim prematurely as the ranges can be -* quite small. By default, do not boost watermarks on discontigmem as in -* many cases very high-order allocations like THP are likely to be -* unsupported and the premature reclaim offsets the advantage of long-term -* fragmentation avoidance. -*/ -int watermark_boost_factor__read_mostly ; -#else int watermark_boost_factor__read_mostly =15000 ; -#endif int watermark_scale_factor =10 ;static unsigned long nr_kernel_pages__initdata ;
_
Andrew Morton
maintainerPATCH 185/192 arch, mm: remove stale mentions of DISCONIGMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: arch, mm: remove stale mentions of DISCONIGMEM
There are several places that mention DISCONIGMEM in comments or have stale code guarded by CONFIG_DISCONTIGMEM.
Remove the dead code and update the comments.
---
arch/ia64/kernel/topology.c | 5 ++--- arch/ia64/mm/numa.c | 5 ++--- arch/mips/include/asm/mmzone.h | 6 ------ arch/mips/mm/init.c | 3 --- arch/nds32/include/asm/memory.h | 6 ------ arch/xtensa/include/asm/page.h | 4 ---- include/linux/gfp.h | 4 ++-- 7 files changed, 6 insertions(+), 27 deletions(-)
arch/ia64/kernel/topology.c renamed from arch/ia64/kernel/topology.c~arch-mm-remove-stale-mentions-of-disconigmem +2 -3
@@ -3,9 +3,8 @@* License. See the file "COPYING" in the main directory of this archive * for more details. * -* This file contains NUMA specific variables and functions which can -* be split away from DISCONTIGMEM and are used on NUMA machines with -* contiguous memory. +* This file contains NUMA specific variables and functions which are used on +* NUMA machines with contiguous memory. * 2002/08/07 Erich Focht <[email protected]> * Populate cpu entries in sysfs for non-numa systems as well * Intel Corporation - Ashok Raj
arch/ia64/mm/numa.c renamed from arch/ia64/mm/numa.c~arch-mm-remove-stale-mentions-of-disconigmem +2 -3
@@ -3,9 +3,8 @@* License. See the file "COPYING" in the main directory of this archive * for more details. * -* This file contains NUMA specific variables and functions which can -* be split away from DISCONTIGMEM and are used on NUMA machines with -* contiguous memory. +* This file contains NUMA specific variables and functions which are used on +* NUMA machines with contiguous memory. * * 2002/08/07 Erich Focht <[email protected]> */
arch/mips/include/asm/mmzone.h renamed from arch/mips/include/asm/mmzone.h~arch-mm-remove-stale-mentions-of-disconigmem +0 -6
@@ -20,10 +20,4 @@#define nid_to_addrbase (nid) 0#endif -#ifdef CONFIG_DISCONTIGMEM - -#define pfn_to_nid (pfn) pa_to_nid((pfn) << PAGE_SHIFT) - -#endif /* CONFIG_DISCONTIGMEM */ -#endif /* _ASM_MMZONE_H_ */
arch/mips/mm/init.c renamed from arch/mips/mm/init.c~arch-mm-remove-stale-mentions-of-disconigmem +0 -3
@@ -454,9 +454,6 @@ void __init mem_init(void)BUILD_BUG_ON (IS_ENABLED (CONFIG_32BIT ) && (_PFN_SHIFT >PAGE_SHIFT ));#ifdef CONFIG_HIGHMEM -#ifdef CONFIG_DISCONTIGMEM -#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet" -#endif max_mapnr = highend_pfn ? highend_pfn : max_low_pfn;#else max_mapnr = max_low_pfn;
arch/nds32/include/asm/memory.h renamed from arch/nds32/include/asm/memory.h~arch-mm-remove-stale-mentions-of-disconigmem +0 -6
@@ -76,18 +76,12 @@* virt_to_page(k) convert a _valid_ virtual address to struct page * * virt_addr_valid(k) indicates whether a virtual address is valid */ -#ifndef CONFIG_DISCONTIGMEM -#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET#define pfn_valid (pfn) ((pfn) >= PHYS_PFN_OFFSET && (pfn) < (PHYS_PFN_OFFSET + max_mapnr))#define virt_to_page (kaddr) (pfn_to_page(__pa(kaddr) >> PAGE_SHIFT))#define virt_addr_valid (kaddr) ((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) -#else /* CONFIG_DISCONTIGMEM */ -#error CONFIG_DISCONTIGMEM is not supported yet. -#endif /* !CONFIG_DISCONTIGMEM */ -#define page_to_phys (page) (page_to_pfn(page) << PAGE_SHIFT)#endif
arch/xtensa/include/asm/page.h renamed from arch/xtensa/include/asm/page.h~arch-mm-remove-stale-mentions-of-disconigmem +0 -4
@@ -192,10 +192,6 @@ static inline unsigned long ___pa(unsign#define pfn_valid (pfn) \ ((pfn) >= ARCH_PFN_OFFSET && ((pfn) - ARCH_PFN_OFFSET) < max_mapnr) -#ifdef CONFIG_DISCONTIGMEM -# error CONFIG_DISCONTIGMEM not supported -#endif -#define virt_to_page (kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)#define page_to_virt (page) __va(page_to_pfn(page) << PAGE_SHIFT)#define virt_addr_valid (kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
include/linux/gfp.h renamed from include/linux/gfp.h~arch-mm-remove-stale-mentions-of-disconigmem +2 -2
@@ -494,8 +494,8 @@ static inline int gfp_zonelist(gfp_t fla* There are two zonelists per node, one for all zones with memory and * one containing just zones from the node the zonelist belongs to. * -* For the normal case of non-DISCONTIGMEM systems the NODE_DATA() gets -* optimized to &contig_page_data at compile-time. +* For the case of non-NUMA systems the NODE_DATA() gets optimized to +* &contig_page_data at compile-time. */ static inline struct zonelist *node_zonelist (int nid,gfp_t flags) {
_
Andrew Morton
maintainerPATCH 186/192 docs: remove description of DISCONTIGMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: docs: remove description of DISCONTIGMEM
Remove description of DISCONTIGMEM from the "Memory Models" document and update VM sysctl description so that it won't mention DISCONIGMEM.
---
Documentation/admin-guide/sysctl/vm.rst | 12 ++--- Documentation/vm/memory-model.rst | 45 ---------------------- 2 files changed, 8 insertions(+), 49 deletions(-)
Documentation/admin-guide/sysctl/vm.rst renamed from Documentation/admin-guide/sysctl/vm.rst~docs-remove-description-of-discontigmem +6 -6
@@ -938,12 +938,12 @@ allocations, THP and hugetlbfs pages. To make it sensible with respect to the watermark_scale_factor parameter, the unit is in fractions of 10,000. The default value of -15,000 on !DISCONTIGMEM configurations means that up to 150% of the high -watermark will be reclaimed in the event of a pageblock being mixed due -to fragmentation. The level of reclaim is determined by the number of -fragmentation events that occurred in the recent past. If this value is -smaller than a pageblock then a pageblocks worth of pages will be reclaimed -(e.g. 2MB on 64-bit x86). A boost factor of 0 will disable the feature. +15,000 means that up to 150% of the high watermark will be reclaimed in the +event of a pageblock being mixed due to fragmentation. The level of reclaim +is determined by the number of fragmentation events that occurred in the +recent past. If this value is smaller than a pageblock then a pageblocks +worth of pages will be reclaimed (e.g. 2MB on 64-bit x86). A boost factor +of 0 will disable the feature. watermark_scale_factor
Documentation/vm/memory-model.rst renamed from Documentation/vm/memory-model.rst~docs-remove-description-of-discontigmem +2 -43
60 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 187/192 mm: replace CONFIG_NEED_MULTIPLE_NODES with CONFIG_NUMA
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: mm: replace CONFIG_NEED_MULTIPLE_NODES with CONFIG_NUMA
After removal of DISCINTIGMEM the NEED_MULTIPLE_NODES and NUMA configuration options are equivalent.
Drop CONFIG_NEED_MULTIPLE_NODES and use CONFIG_NUMA instead.
Done with
$ sed -i 's/CONFIG_NEED_MULTIPLE_NODES/CONFIG_NUMA/' \ $(git grep -wl CONFIG_NEED_MULTIPLE_NODES) $ sed -i 's/NEED_MULTIPLE_NODES/NUMA/' \ $(git grep -wl NEED_MULTIPLE_NODES)
with manual tweaks afterwards.
---
arch/arm64/Kconfig | 2 +- arch/ia64/Kconfig | 2 +- arch/mips/Kconfig | 2 +- arch/mips/include/asm/mmzone.h | 2 +- arch/mips/include/asm/page.h | 2 +- arch/mips/mm/init.c | 4 ++-- arch/powerpc/Kconfig | 2 +- arch/powerpc/include/asm/mmzone.h | 4 ++-- arch/powerpc/kernel/setup_64.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/kexec/core.c | 4 ++-- arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/mem.c | 4 ++-- arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/sh/include/asm/mmzone.h | 4 ++-- arch/sh/kernel/topology.c | 2 +- arch/sh/mm/Kconfig | 2 +- arch/sh/mm/init.c | 2 +- arch/sparc/Kconfig | 2 +- arch/sparc/include/asm/mmzone.h | 4 ++-- arch/sparc/kernel/smp_64.c | 2 +- arch/sparc/mm/init_64.c | 12 ++++++------ arch/x86/Kconfig | 2 +- arch/x86/kernel/setup_percpu.c | 6 +++--- arch/x86/mm/init_32.c | 4 ++-- include/asm-generic/topology.h | 2 +- include/linux/memblock.h | 6 +++--- include/linux/mm.h | 4 ++-- include/linux/mmzone.h | 6 +++--- kernel/crash_core.c | 2 +- mm/Kconfig | 9 --------- mm/memblock.c | 8 ++++---- mm/memory.c | 3 +-- mm/page_alloc.c | 6 +++--- mm/sparse.c | 2 +- 36 files changed, 59 insertions(+), 69 deletions(-)
arch/arm64/Kconfig renamed from arch/arm64/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -1035,7 +1035,7 @@ config NODES_SHIFT int "Maximum NUMA Nodes (as a power of 2)" range 1 10 default "4" - depends on NEED_MULTIPLE_NODES + depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables.
arch/ia64/Kconfig renamed from arch/ia64/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -302,7 +302,7 @@ config NODES_SHIFT int "Max num nodes shift(3-10)" range 3 10 default "10" - depends on NEED_MULTIPLE_NODES + depends on NUMA help This option specifies the maximum number of nodes in your SSI system. MAX_NUMNODES will be 2^(This value).
arch/mips/include/asm/mmzone.h renamed from arch/mips/include/asm/mmzone.h~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -8,7 +8,7 @@#include <asm/page.h> -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA # include <mmzone.h> #endif
arch/mips/include/asm/page.h renamed from arch/mips/include/asm/page.h~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -239,7 +239,7 @@ static inline int pfn_valid(unsigned lon/* pfn_valid is defined in linux/mmzone.h */ -#elif defined(CONFIG_NEED_MULTIPLE_NODES ) +#elif defined(CONFIG_NUMA )#define pfn_valid (pfn) \ ({ \
arch/mips/Kconfig renamed from arch/mips/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -2867,7 +2867,7 @@ config RANDOMIZE_BASE_MAX_OFFSET config NODES_SHIFT int default "6" - depends on NEED_MULTIPLE_NODES + depends on NUMA config HW_PERF_EVENTS bool "Enable hardware performance counter support for perf events"
arch/mips/mm/init.c renamed from arch/mips/mm/init.c~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -394,7 +394,7 @@ void maar_init(void) } } -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA void __init paging_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ];
@@ -473,7 +473,7 @@ void __init mem_init(void) 0x80000000 - 4, KCORE_TEXT); #endif } -#endif /* !CONFIG_NEED_MULTIPLE_NODES */ +#endif /* !CONFIG_NUMA */ void free_init_pages (const char *what,unsigned long begin,unsigned long end) {
arch/powerpc/include/asm/mmzone.h renamed from arch/powerpc/include/asm/mmzone.h~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -18,7 +18,7 @@* flags field of the struct page */ -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA extern struct pglist_data *node_data[];/*
@@ -41,7 +41,7 @@ u64 memory_hotplug_max(void);#else #define memory_hotplug_max () memblock_end_of_DRAM() -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ #ifdef CONFIG_FA_DUMP #define __HAVE_ARCH_RESERVED_KERNEL_PAGES#endif
arch/powerpc/Kconfig renamed from arch/powerpc/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -671,7 +671,7 @@ config NODES_SHIFT int default "8" if PPC64 default "4" - depends on NEED_MULTIPLE_NODES + depends on NUMA config USE_PERCPU_NUMA_NODE_ID def_bool y
arch/powerpc/kernel/setup_64.c renamed from arch/powerpc/kernel/setup_64.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -788,7 +788,7 @@ static void * __init pcpu_alloc_bootmem( size_t align) {const unsigned long goal =__pa (MAX_DMA_ADDRESS ); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int node =early_cpu_to_node (cpu);void *ptr;
arch/powerpc/kernel/smp.c renamed from arch/powerpc/kernel/smp.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -1047,7 +1047,7 @@ void __init smp_prepare_cpus(unsigned inzalloc_cpumask_var_node (&per_cpu (cpu_coregroup_map, cpu),GFP_KERNEL ,cpu_to_node (cpu)); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA /* * numa_node_id() works after this. */
arch/powerpc/kexec/core.c renamed from arch/powerpc/kexec/core.c~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -68,11 +68,11 @@ void machine_kexec_cleanup(struct kimagevoid arch_crash_save_vmcoreinfo (void ) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA VMCOREINFO_SYMBOL (node_data);VMCOREINFO_LENGTH (node_data,MAX_NUMNODES );#endif -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA VMCOREINFO_SYMBOL (contig_page_data);#endif #if defined(CONFIG_PPC64 ) && defined(CONFIG_SPARSEMEM_VMEMMAP )
arch/powerpc/mm/Makefile renamed from arch/powerpc/mm/Makefile~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -13,7 +13,7 @@ obj-y := fault.o mem.o pgtable.o mmap obj-$(CONFIG_PPC_MMU_NOHASH) += nohash/ obj-$(CONFIG_PPC_BOOK3S_32) += book3s32/ obj-$(CONFIG_PPC_BOOK3S_64) += book3s64/ -obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o +obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_PPC_MM_SLICES) += slice.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o
arch/powerpc/mm/mem.c renamed from arch/powerpc/mm/mem.c~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -127,7 +127,7 @@ void __ref arch_remove_memory(int nid, u }#endif -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA void __init mem_topology_setup (void ) { max_low_pfn = max_pfn =memblock_end_of_DRAM () >>PAGE_SHIFT ;
@@ -162,7 +162,7 @@ static int __init mark_nonram_nosave(voi return 0; } -#else /* CONFIG_NEED_MULTIPLE_NODES */ +#else /* CONFIG_NUMA */ static int __init mark_nonram_nosave (void ) {return 0 ;
arch/riscv/Kconfig renamed from arch/riscv/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -332,7 +332,7 @@ config NODES_SHIFT int "Maximum NUMA Nodes (as a power of 2)" range 1 10 default "2" - depends on NEED_MULTIPLE_NODES + depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables.
arch/s390/Kconfig renamed from arch/s390/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -475,7 +475,7 @@ config NUMA config NODES_SHIFT int - depends on NEED_MULTIPLE_NODES + depends on NUMA default "1" config SCHED_SMT
arch/sh/include/asm/mmzone.h renamed from arch/sh/include/asm/mmzone.h~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -2,7 +2,7 @@#ifndef __ASM_SH_MMZONE_H#define __ASM_SH_MMZONE_H -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA #include <linux/numa.h> extern struct pglist_data *node_data[];
@@ -31,7 +31,7 @@ static inline void setup_bootmem_node(int nid, unsigned long start, unsigned long end) { } -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ /* Platform specific mem init */ void __init plat_mem_setup (void );
arch/sh/kernel/topology.c renamed from arch/sh/kernel/topology.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -46,7 +46,7 @@ static int __init topology_init(void) {int i, ret; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA for_each_online_node (i)register_one_node (i);#endif
arch/sh/mm/init.c renamed from arch/sh/mm/init.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -211,7 +211,7 @@ void __init allocate_pgdat(unsigned intget_pfn_range_for_nid (nid, &start_pfn, &end_pfn); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA NODE_DATA(nid) = memblock_alloc_try_nid( sizeof(struct pglist_data), SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
arch/sh/mm/Kconfig renamed from arch/sh/mm/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -120,7 +120,7 @@ config NODES_SHIFT int default "3" if CPU_SUBTYPE_SHX3 default "1" - depends on NEED_MULTIPLE_NODES + depends on NUMA config ARCH_FLATMEM_ENABLE def_bool y
arch/sparc/include/asm/mmzone.h renamed from arch/sparc/include/asm/mmzone.h~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -2,7 +2,7 @@#ifndef _SPARC64_MMZONE_H#define _SPARC64_MMZONE_H -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA #include <linux/cpumask.h>
@@ -13,6 +13,6 @@ extern struct pglist_data *node_data[];extern int numa_cpu_lookup_table[];extern cpumask_t numa_cpumask_lookup_table[]; -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ #endif /* _SPARC64_MMZONE_H */
arch/sparc/Kconfig renamed from arch/sparc/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -265,7 +265,7 @@ config NODES_SHIFT int "Maximum NUMA Nodes (as a power of 2)" range 4 5 if SPARC64 default "5" - depends on NEED_MULTIPLE_NODES + depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables.
arch/sparc/kernel/smp_64.c renamed from arch/sparc/kernel/smp_64.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -1546,7 +1546,7 @@ static void * __init pcpu_alloc_bootmem( size_t align) {const unsigned long goal =__pa (MAX_DMA_ADDRESS ); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int node =cpu_to_node (cpu);void *ptr;
arch/sparc/mm/init_64.c renamed from arch/sparc/mm/init_64.c~mm-replace-config_need_multiple_nodes-with-config_numa +6 -6
@@ -903,7 +903,7 @@ struct node_mem_mask {static struct node_mem_mask node_masks[MAX_NUMNODES ];static int num_node_masks; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA struct mdesc_mlgroup { u64 node;
@@ -1059,7 +1059,7 @@ static void __init allocate_node_data(in {struct pglist_data *p;unsigned long start_pfn, end_pfn; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA NODE_DATA (nid) =memblock_alloc_node (sizeof (struct pglist_data ),SMP_CACHE_BYTES , nid);
@@ -1080,7 +1080,7 @@ static void __init allocate_node_data(instatic void init_node_masks_nonnuma (void ) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int i;#endif
@@ -1090,7 +1090,7 @@ static void init_node_masks_nonnuma(void node_masks[0 ].match =0 ; num_node_masks =1 ; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA for (i =0 ; i <NR_CPUS ; i++) numa_cpu_lookup_table[i] =0 ;
@@ -1098,7 +1098,7 @@ static void init_node_masks_nonnuma(void#endif } -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA struct pglist_data *node_data[MAX_NUMNODES ];EXPORT_SYMBOL (numa_cpu_lookup_table);
@@ -2487,7 +2487,7 @@ int page_in_phys_avail(unsigned long padstatic void __init register_page_bootmem_info (void ) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int i;for_each_online_node (i)
arch/x86/Kconfig renamed from arch/x86/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -1597,7 +1597,7 @@ config NODES_SHIFT default "10" if MAXSMP default "6" if X86_64 default "3" - depends on NEED_MULTIPLE_NODES + depends on NUMA help Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables.
arch/x86/kernel/setup_percpu.c renamed from arch/x86/kernel/setup_percpu.c~mm-replace-config_need_multiple_nodes-with-config_numa +3 -3
@@ -66,7 +66,7 @@ EXPORT_SYMBOL(__per_cpu_offset);*/ static bool __init pcpu_need_numa (void ) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA pg_data_t *last =NULL ;unsigned int cpu;
@@ -101,7 +101,7 @@ static void * __init pcpu_alloc_bootmem( unsigned long align) {const unsigned long goal =__pa (MAX_DMA_ADDRESS ); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int node =early_cpu_to_node (cpu);void *ptr;
@@ -140,7 +140,7 @@ static void __init pcpu_fc_free(void *ptstatic int __init pcpu_cpu_distance (unsigned int from,unsigned int to) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA if (early_cpu_to_node (from) ==early_cpu_to_node (to))return LOCAL_DISTANCE ;else
arch/x86/mm/init_32.c renamed from arch/x86/mm/init_32.c~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -651,7 +651,7 @@ void __init find_low_pfn_range(void)highmem_pfn_init (); } -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA void __init initmem_init (void ) {#ifdef CONFIG_HIGHMEM
@@ -677,7 +677,7 @@ void __init initmem_init(void) setup_bootmem_allocator(); } -#endif /* !CONFIG_NEED_MULTIPLE_NODES */ +#endif /* !CONFIG_NUMA */ void __init setup_bootmem_allocator (void ) {
include/asm-generic/topology.h renamed from include/asm-generic/topology.h~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -45,7 +45,7 @@#endif #ifndef cpumask_of_node -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA #define cpumask_of_node (node) ((node) == 0 ? cpu_online_mask : cpu_none_mask)#else #define cpumask_of_node (node) ((void)(node), cpu_online_mask)
include/linux/memblock.h renamed from include/linux/memblock.h~mm-replace-config_need_multiple_nodes-with-config_numa +3 -3
@@ -50,7 +50,7 @@ struct memblock_region {phys_addr_t base;phys_addr_t size;enum memblock_flags flags; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int nid;#endif };
@@ -347,7 +347,7 @@ int __init deferred_page_init_max_threadint memblock_set_node (phys_addr_t base,phys_addr_t size,struct memblock_type *type,int nid); -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA static inline void memblock_set_region_node (struct memblock_region *r,int nid) { r->nid = nid;
@@ -366,7 +366,7 @@ static inline int memblock_get_region_no {return 0 ; } -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ /* Flags for memblock allocation APIs */ #define MEMBLOCK_ALLOC_ANYWHERE (~(phys_addr_t)0)
include/linux/mm.h renamed from include/linux/mm.h~mm-replace-config_need_multiple_nodes-with-config_numa +2 -2
@@ -46,7 +46,7 @@ extern int sysctl_page_lock_unfairness;void init_mm_internals (void ); -#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */ +#ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */ extern unsigned long max_mapnr;static inline void set_max_mapnr (unsigned long limit)
@@ -2460,7 +2460,7 @@ extern void get_pfn_range_for_nid(unsign unsigned long *start_pfn, unsigned long *end_pfn);extern unsigned long find_min_pfn_with_active_regions (void ); -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA static inline int early_pfn_to_nid (unsigned long pfn) {return 0 ;
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-replace-config_need_multiple_nodes-with-config_numa +3 -3
@@ -1043,17 +1043,17 @@ extern int percpu_pagelist_high_fractionextern char numa_zonelist_order[];#define NUMA_ZONELIST_ORDER_LEN 16 -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA extern struct pglist_data contig_page_data;#define NODE_DATA (nid) (&contig_page_data)#define NODE_MEM_MAP (nid) mem_map -#else /* CONFIG_NEED_MULTIPLE_NODES */ +#else /* CONFIG_NUMA */ #include <asm/mmzone.h> -#endif /* !CONFIG_NEED_MULTIPLE_NODES */ +#endif /* !CONFIG_NUMA */ extern struct pglist_data *first_online_pgdat (void );extern struct pglist_data *next_online_pgdat (struct pglist_data *pgdat);
kernel/crash_core.c renamed from kernel/crash_core.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -455,7 +455,7 @@ static int __init crash_save_vmcoreinfo_VMCOREINFO_SYMBOL (_stext);VMCOREINFO_SYMBOL (vmap_area_list); -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA VMCOREINFO_SYMBOL (mem_map);VMCOREINFO_SYMBOL (contig_page_data);#endif
mm/Kconfig renamed from mm/Kconfig~mm-replace-config_need_multiple_nodes-with-config_numa +0 -9
@@ -60,15 +60,6 @@ config FLAT_NODE_MEM_MAP depends on !SPARSEMEM # -# Both the NUMA code and DISCONTIGMEM use arrays of pg_data_t's -# to represent different areas of memory. This variable allows -# those dependencies to exist individually. -# -config NEED_MULTIPLE_NODES - def_bool y - depends on NUMA - -# # SPARSEMEM_EXTREME (which is the default) does some bootmem # allocations when sparse_init() is called. If this cannot # be done on your architecture, select this option. However,
mm/memblock.c renamed from mm/memblock.c~mm-replace-config_need_multiple_nodes-with-config_numa +4 -4
@@ -92,7 +92,7 @@* system initialization completes. */ -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA struct pglist_data __refdata contig_page_data;EXPORT_SYMBOL (contig_page_data);#endif
@@ -607,7 +607,7 @@ repeat:* area, insert that portion. */ if (rbase > base) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA WARN_ON (nid !=memblock_get_region_node (rgn));#endif WARN_ON (flags != rgn->flags);
@@ -1205,7 +1205,7 @@ void __init_memblock __next_mem_pfn_rang int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,struct memblock_type *type,int nid) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int start_rgn, end_rgn;int i, ret;
@@ -1849,7 +1849,7 @@ static void __init_memblock memblock_dum size = rgn->size; end = base + size -1 ; flags = rgn->flags; -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA if (memblock_get_region_node (rgn) !=MAX_NUMNODES )snprintf (nid_buf,sizeof (nid_buf)," on node %d" ,memblock_get_region_node (rgn));
mm/memory.c renamed from mm/memory.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -2
@@ -90,8 +90,7 @@#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif -#ifndef CONFIG_NEED_MULTIPLE_NODES -/* use the per-pgdat data instead for discontigmem - mbligh */ +#ifndef CONFIG_NUMA unsigned long max_mapnr;EXPORT_SYMBOL (max_mapnr);
mm/page_alloc.c renamed from mm/page_alloc.c~mm-replace-config_need_multiple_nodes-with-config_numa +3 -3
@@ -1634,7 +1634,7 @@ void __free_pages_core(struct page *page__free_pages_ok (page, order,FPI_TO_TAIL |FPI_SKIP_KASAN_POISON ); } -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA /* * During memory init memblocks map pfns to nids. The search is expensive and
@@ -1684,7 +1684,7 @@ int __meminit early_pfn_to_nid(unsigned return nid; } -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ void __init memblock_free_pages (struct page *page,unsigned long pfn,unsigned int order)
@@ -7438,7 +7438,7 @@ static void __ref alloc_node_mem_map(strpr_debug ("%s: node %d, pgdat %08lx, node_mem_map %08lx\n" , __func__, pgdat->node_id, (unsigned long )pgdat, (unsigned long )pgdat->node_mem_map); -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA /* * With no DISCONTIG, the global mem_map is just set as node 0's */
mm/sparse.c renamed from mm/sparse.c~mm-replace-config_need_multiple_nodes-with-config_numa +1 -1
@@ -346,7 +346,7 @@ size_t mem_section_usage_size(void)static inline phys_addr_t pgdat_to_phys (struct pglist_data *pgdat) { -#ifndef CONFIG_NEED_MULTIPLE_NODES +#ifndef CONFIG_NUMA return __pa_symbol (pgdat);#else return __pa (pgdat);
_
Andrew Morton
maintainerPATCH 188/192 mm: replace CONFIG_FLAT_NODE_MEM_MAP with CONFIG_FLATMEM
In reply to Andrew MortonFrom: Mike Rapoport <[email protected]>
Subject: mm: replace CONFIG_FLAT_NODE_MEM_MAP with CONFIG_FLATMEM
After removal of the DISCONTIGMEM memory model the FLAT_NODE_MEM_MAP configuration option is equivalent to FLATMEM.
Drop CONFIG_FLAT_NODE_MEM_MAP and use CONFIG_FLATMEM instead.
---
include/linux/mmzone.h | 4 ++-- kernel/crash_core.c | 2 +- mm/Kconfig | 4 ---- mm/page_alloc.c | 6 +++--- mm/page_ext.c | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-replace-config_flat_node_mem_map-with-config_flatmem +2 -2
@@ -788,7 +788,7 @@ typedef struct pglist_data {struct zonelist node_zonelists[MAX_ZONELISTS ];int nr_zones;/* number of populated zones in this node */ -#ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */ +#ifdef CONFIG_FLATMEM /* means !SPARSEMEM */ struct page *node_mem_map;#ifdef CONFIG_PAGE_EXTENSION struct page_ext *node_page_ext;
@@ -878,7 +878,7 @@ typedef struct pglist_data {#define node_present_pages (nid) (NODE_DATA(nid)->node_present_pages)#define node_spanned_pages (nid) (NODE_DATA(nid)->node_spanned_pages) -#ifdef CONFIG_FLAT_NODE_MEM_MAP +#ifdef CONFIG_FLATMEM #define pgdat_page_nr (pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr))#else #define pgdat_page_nr (pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr))
kernel/crash_core.c renamed from kernel/crash_core.c~mm-replace-config_flat_node_mem_map-with-config_flatmem +1 -1
@@ -484,7 +484,7 @@ static int __init crash_save_vmcoreinfo_VMCOREINFO_OFFSET (page, compound_head);VMCOREINFO_OFFSET (pglist_data, node_zones);VMCOREINFO_OFFSET (pglist_data, nr_zones); -#ifdef CONFIG_FLAT_NODE_MEM_MAP +#ifdef CONFIG_FLATMEM VMCOREINFO_OFFSET (pglist_data, node_mem_map);#endif VMCOREINFO_OFFSET (pglist_data, node_start_pfn);
mm/Kconfig renamed from mm/Kconfig~mm-replace-config_flat_node_mem_map-with-config_flatmem +0 -4
@@ -55,10 +55,6 @@ config FLATMEM def_bool y depends on !SPARSEMEM || FLATMEM_MANUAL -config FLAT_NODE_MEM_MAP - def_bool y - depends on !SPARSEMEM - # # SPARSEMEM_EXTREME (which is the default) does some bootmem # allocations when sparse_init() is called. If this cannot
mm/page_alloc.c renamed from mm/page_alloc.c~mm-replace-config_flat_node_mem_map-with-config_flatmem +3 -3
@@ -6547,7 +6547,7 @@ static void __meminit zone_init_free_lis } } -#if !defined(CONFIG_FLAT_NODE_MEM_MAP ) +#if !defined(CONFIG_FLATMEM )/* * Only struct pages that correspond to ranges defined by memblock.memory * are zeroed and initialized by going through __init_single_page() during
@@ -7403,7 +7403,7 @@ static void __init free_area_init_core(s } } -#ifdef CONFIG_FLAT_NODE_MEM_MAP +#ifdef CONFIG_FLATMEM static void __ref alloc_node_mem_map (struct pglist_data *pgdat) {unsigned long __maybe_unused start =0 ;
@@ -7451,7 +7451,7 @@ static void __ref alloc_node_mem_map(str }#else static void __ref alloc_node_mem_map (struct pglist_data *pgdat) { } -#endif /* CONFIG_FLAT_NODE_MEM_MAP */ +#endif /* CONFIG_FLATMEM */ #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range (pg_data_t *pgdat)
mm/page_ext.c renamed from mm/page_ext.c~mm-replace-config_flat_node_mem_map-with-config_flatmem +1 -1
@@ -191,7 +191,7 @@ fail: panic("Out of memory"); } -#else /* CONFIG_FLAT_NODE_MEM_MAP */ +#else /* CONFIG_FLATMEM */ struct page_ext *lookup_page_ext (const struct page *page) {
_
Andrew Morton
maintainerPATCH 189/192 mm/page_alloc: allow high-order pages to be stored on the per-cpu lists
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: allow high-order pages to be stored on the per-cpu lists
The per-cpu page allocator (PCP) only stores order-0 pages. This means that all THP and "cheap" high-order allocations including SLUB contends on the zone->lock. This patch extends the PCP allocator to store THP and "cheap" high-order pages. Note that struct per_cpu_pages increases in size to 256 bytes (4 cache lines) on x86-64.
Note that this is not necessarily a universal performance win because of how it is implemented. High-order pages can cause pcp->high to be exceeded prematurely for lower-orders so for example, a large number of THP pages being freed could release order-0 pages from the PCP lists. Hence, much depends on the allocation/free pattern as observed by a single CPU to determine if caching helps or hurts a particular workload.
That said, basic performance testing passed. The following is a netperf UDP_STREAM test which hits the relevant patches as some of the network allocations are high-order.
netperf-udp
5.13.0-rc2 5.13.0-rc2
mm-pcpburst-v3r4 mm-pcphighorder-v1r7
Hmean send-64 261.46 ( 0.00%) 266.30 * 1.85%*
Hmean send-128 516.35 ( 0.00%) 536.78 * 3.96%*
Hmean send-256 1014.13 ( 0.00%) 1034.63 * 2.02%*
Hmean send-1024 3907.65 ( 0.00%) 4046.11 * 3.54%*
Hmean send-2048 7492.93 ( 0.00%) 7754.85 * 3.50%*
Hmean send-3312 11410.04 ( 0.00%) 11772.32 * 3.18%*
Hmean send-4096 13521.95 ( 0.00%) 13912.34 * 2.89%*
Hmean send-8192 21660.50 ( 0.00%) 22730.72 * 4.94%*
Hmean send-16384 31902.32 ( 0.00%) 32637.50 * 2.30%*Functionally, a patch like this is necessary to make bulk allocation of high-order pages work with similar performance to order-0 bulk allocations. The bulk allocator is not updated in this series as it would have to be determined by bulk allocation users how they want to track the order of pages allocated with the bulk allocator.
---
include/linux/mmzone.h | 20 ++++ mm/internal.h | 2 mm/page_alloc.c | 169 ++++++++++++++++++++++++++++----------- mm/swap.c | 2 4 files changed, 144 insertions(+), 49 deletions(-)
include/linux/mmzone.h renamed from include/linux/mmzone.h~mm-page_alloc-allow-high-order-pages-to-be-stored-on-the-per-cpu-lists +19 -1
@@ -333,6 +333,24 @@ enum zone_watermarks { NR_WMARK }; +/* +* One per migratetype for each PAGE_ALLOC_COSTLY_ORDER plus one additional +* for pageblock size for THP if configured. +*/ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define NR_PCP_THP 1 +#else +#define NR_PCP_THP 0 +#endif +#define NR_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1 + NR_PCP_THP)) + +/* +* Shift to encode migratetype and order in the same integer, with order +* in the least significant bits. +*/ +#define NR_PCP_ORDER_WIDTH 8 +#define NR_PCP_ORDER_MASK ((1<<NR_PCP_ORDER_WIDTH) - 1) +#define min_wmark_pages (z) (z->_watermark[WMARK_MIN] + z->watermark_boost)#define low_wmark_pages (z) (z->_watermark[WMARK_LOW] + z->watermark_boost)#define high_wmark_pages (z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
@@ -349,7 +367,7 @@ struct per_cpu_pages {#endif /* Lists of pages, one per migrate type stored on the pcp-lists */ -struct list_head lists[MIGRATE_PCPTYPES ]; +struct list_head lists[NR_PCP_LISTS ]; };struct per_cpu_zonestat {
mm/internal.h renamed from mm/internal.h~mm-page_alloc-allow-high-order-pages-to-be-stored-on-the-per-cpu-lists +1 -1
@@ -203,7 +203,7 @@ extern void post_alloc_hook(struct page gfp_t gfp_flags);extern int user_min_free_kbytes; -extern void free_unref_page (struct page *page); +extern void free_unref_page (struct page *page,unsigned int order);extern void free_unref_page_list (struct list_head *list);extern void zone_pcp_update (struct zone *zone,int cpu_online);
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-allow-high-order-pages-to-be-stored-on-the-per-cpu-lists +123 -46
373 lines not on this page · raw message
mm/swap.c renamed from mm/swap.c~mm-page_alloc-allow-high-order-pages-to-be-stored-on-the-per-cpu-lists +1 -1
@@ -95,7 +95,7 @@ static void __put_single_page(struct pag {__page_cache_release (page);mem_cgroup_uncharge (page); -free_unref_page (page); +free_unref_page (page,0 ); }static void __put_compound_page (struct page *page)
_
Andrew Morton
maintainerPATCH 190/192 mm/page_alloc: split pcp->high across all online CPUs for cpuless nodes
In reply to Andrew MortonFrom: Mel Gorman <[email protected]>
Subject: mm/page_alloc: split pcp->high across all online CPUs for cpuless nodes
Dave Hansen reported the following about Feng Tang's tests on a machine with persistent memory onlined as a DRAM-like device.
Feng Tang tossed these on a "Cascade Lake" system with 96 threads and ~512G of persistent memory and 128G of DRAM. The PMEM is in "volatile use" mode and being managed via the buddy just like the normal RAM.
The PMEM zones are big ones:
present 65011712 = 248 G
high 134595 = 525 MThe PMEM nodes, of course, don't have any CPUs in them.
With your series, the pcp->high value per-cpu is 69584 pages or about 270MB per CPU. Scaled up by the 96 CPU threads, that's ~26GB of worst-case memory in the pcps per zone, or roughly 10% of the size of the zone.
This should not cause a problem as such although it could trigger reclaim due to pages being stored on per-cpu lists for CPUs remote to a node. It is not possible to treat cpuless nodes exactly the same as normal nodes but the worst-case scenario can be mitigated by splitting pcp->high across all online CPUs for cpuless memory nodes.
---
mm/page_alloc.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-)
mm/page_alloc.c renamed from mm/page_alloc.c~mm-page_alloc-split-pcp-high-across-all-online-cpus-for-cpuless-nodes +8 -4
@@ -6790,7 +6790,7 @@ static int zone_highsize(struct zone *zo {#ifdef CONFIG_MMU int high; -int nr_local_cpus; +int nr_split_cpus;unsigned long total_pages;if (!percpu_pagelist_high_fraction) {
@@ -6813,10 +6813,14 @@ static int zone_highsize(struct zone *zo* Split the high value across all online CPUs local to the zone. Note * that early in boot that CPUs may not be online yet and that during * CPU hotplug that the cpumask is not yet updated when a CPU is being -* onlined. +* onlined. For memory nodes that have no CPUs, split pcp->high across +* all online CPUs to mitigate the risk that reclaim is triggered +* prematurely due to pages stored on pcp lists. */ - nr_local_cpus =max (1U ,cpumask_weight (cpumask_of_node (zone_to_nid (zone)))) + cpu_online; - high = total_pages / nr_local_cpus; + nr_split_cpus =cpumask_weight (cpumask_of_node (zone_to_nid (zone))) + cpu_online; +if (!nr_split_cpus) + nr_split_cpus =num_online_cpus (); + high = total_pages / nr_split_cpus;/* * Ensure high is at least batch*4. The multiple is based on the
_
Andrew Morton
maintainerPATCH 191/192 mm,hwpoison: send SIGBUS with error virutal address
In reply to Andrew MortonFrom: Naoya Horiguchi <[email protected]>
Subject: mm,hwpoison: send SIGBUS with error virutal address
Now an action required MCE in already hwpoisoned address surely sends a SIGBUS to current process, but the SIGBUS doesn't convey error virtual address. That's not optimal for hwpoison-aware applications.
To fix the issue, make memory_failure() call kill_accessing_process(), that does pagetable walk to find the error virtual address. It could find multiple virtual addresses for the same error page, and it seems hard to tell which virtual address is correct one. But that's rare and sending incorrect virtual address could be better than no address. So let's report the first found virtual address for now.
---
arch/x86/kernel/cpu/mce/core.c | 13 ++ include/linux/swapops.h | 5 + mm/memory-failure.c | 150 ++++++++++++++++++++++++++++++- 3 files changed, 165 insertions(+), 3 deletions(-)
arch/x86/kernel/cpu/mce/core.c renamed from arch/x86/kernel/cpu/mce/core.c~mmhwpoison-send-sigbus-with-error-virutal-address +11 -2
@@ -1257,19 +1257,28 @@ static void kill_me_maybe(struct callbac { struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);int flags =MF_ACTION_REQUIRED ; +int ret;pr_err ("Uncorrected hardware memory error in user-access at %llx" , p->mce_addr);if (!p->mce_ripv) flags |=MF_MUST_KILL ; -if (!memory_failure (p->mce_addr >>PAGE_SHIFT , flags) && - !(p->mce_kflags &MCE_IN_KERNEL_COPYIN )) { + ret =memory_failure (p->mce_addr >>PAGE_SHIFT , flags); +if (!ret && !(p->mce_kflags &MCE_IN_KERNEL_COPYIN )) {set_mce_nospec (p->mce_addr >>PAGE_SHIFT , p->mce_whole_page);sync_core ();return ; } +/* +* -EHWPOISON from memory_failure() means that it already sent SIGBUS +* to the current process with the proper error info, so no need to +* send SIGBUS here again. +*/ +if (ret == -EHWPOISON ) +return ; +if (p->mce_vaddr != (void __user *)-1l ) {force_sig_mceerr (BUS_MCEERR_AR , p->mce_vaddr,PAGE_SHIFT ); }else {
include/linux/swapops.h renamed from include/linux/swapops.h~mmhwpoison-send-sigbus-with-error-virutal-address +5 -0
@@ -330,6 +330,11 @@ static inline int is_hwpoison_entry(swp_return swp_type (entry) ==SWP_HWPOISON ; } +static inline unsigned long hwpoison_entry_to_pfn (swp_entry_t entry) +{ +return swp_offset (entry); +} +static inline void num_poisoned_pages_inc (void ) {atomic_long_inc (&num_poisoned_pages);
mm/memory-failure.c renamed from mm/memory-failure.c~mmhwpoison-send-sigbus-with-error-virutal-address +149 -1
174 lines not on this page · raw message
_
Andrew Morton
maintainerPATCH 192/192 mm,hwpoison: make get_hwpoison_page() call get_any_page()
In reply to Andrew MortonFrom: Naoya Horiguchi <[email protected]>
Subject: mm,hwpoison: make get_hwpoison_page() call get_any_page()
__get_hwpoison_page() could fail to grab refcount by some race condition, so it's helpful if we can handle it by retrying. We already have retry logic, so make get_hwpoison_page() call get_any_page() when called from memory_failure().
As a result, get_hwpoison_page() can return negative values (i.e. error code), so some callers are also changed to handle error cases. soft_offline_page() does nothing for -EBUSY because that's enough and users in userspace can easily handle it. unpoison_memory() is also unchanged because it's broken and need thorough fixes (will be done later).
---
mm/hugetlb.c | 2 mm/memory-failure.c | 194 +++++++++++++++++++++++------------------- 2 files changed, 111 insertions(+), 85 deletions(-)
mm/hugetlb.c renamed from mm/hugetlb.c~mmhwpoison-make-get_hwpoison_page-call-get_any_page +2 -0
@@ -5938,6 +5938,8 @@ int get_hwpoison_huge_page(struct page * *hugetlb = true;if (HPageFreed (page) ||HPageMigratable (page)) ret =get_page_unless_zero (page); +else + ret = -EBUSY ; }spin_unlock_irq (&hugetlb_lock);return ret;
mm/memory-failure.c renamed from mm/memory-failure.c~mmhwpoison-make-get_hwpoison_page-call-get_any_page +109 -85
256 lines not on this page · raw message
_