Deferred page init improvements
Subsystems (7)
v2 cover Deferred page init improvements
This patchset is essentially a refactor of the page initialization logic that is meant to provide for better code reuse while providing a significant improvement in deferred page initialization performance.
In my testing I have seen a 60% reduction in the time needed for deferred memory initialization on two different x86_64 based test systems I have. In addition this provides a very slight improvement for the hotplug memory initialization, although the improvement doesn't exceed 5% from what I can tell and that is to be expected since most of the changes related to hotplug init are mostly just code clean-up to allow for reuse.
The biggest gains of this patchset come from not having to test each pfn multiple times to see if it is valid and if it is actually a part of the node being initialized.
v1->v2:
Fixed build issue on PowerPC due to page struct size being 56 Added new patch that removed __SetPageReserved call for hotplug
---
Alexander Duyck (6):
mm: Use mm_zero_struct_page from SPARC on all 64b architectures
mm: Drop meminit_pfn_in_nid as it is redundant
mm: Use memblock/zone specific iterator for handling deferred page init
mm: Do not set reserved flag for hotplug memory
mm: Move hot-plug specific memory init into separate functions and optimize
mm: Use common iterator for deferred_init_pages and deferred_free_pages
arch/sparc/include/asm/pgtable_64.h | 30 -- include/linux/memblock.h | 58 ++++ include/linux/mm.h | 34 ++ mm/memblock.c | 63 ++++ mm/page_alloc.c | 552 +++++++++++++++++++++-------------- 5 files changed, 483 insertions(+), 254 deletions(-)
--
PATCH v2 1/6 mm: Use mm_zero_struct_page from SPARC on all 64b architectures
This change makes it so that we use the same approach that was already in use on Sparc on all the archtectures that support a 64b long.
This is mostly motivated by the fact that 8 to 10 store/move instructions are likely always going to be faster than having to call into a function that is not specialized for handling page init.
An added advantage to doing it this way is that the compiler can get away with combining writes in the __init_single_page call. As a result the memset call will be reduced to only about 4 write operations, or at least that is what I am seeing with GCC 6.2 as the flags, LRU poitners, and count/mapcount seem to be cancelling out at least 4 of the 8 assignments on my system.
One change I had to make to the function was to reduce the minimum page size to 56 to support some powerpc64 configurations.
--- arch/sparc/include/asm/pgtable_64.h | 30 ------------------------------ include/linux/mm.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-)
arch/sparc/include/asm/pgtable_64.h +0 -30
@@ -231,36 +231,6 @@extern struct page *mem_map_zero;#define ZERO_PAGE (vaddr) (mem_map_zero) -/* This macro must be updated when the size of struct page grows above 80 -* or reduces below 64. -* The idea that compiler optimizes out switch() statement, and only -* leaves clrx instructions -*/ -#define mm_zero_struct_page (pp) do { \ - unsigned long *_pp = (void *)(pp); \ - \ -/* Check that struct page is either 64, 72, or 80 bytes */ \ -BUILD_BUG_ON (sizeof (struct page ) &7 ); \ -BUILD_BUG_ON (sizeof (struct page ) <64 ); \ -BUILD_BUG_ON (sizeof (struct page ) >80 ); \ - \ -switch (sizeof (struct page )) { \ -case 80 : \ - _pp[9 ] =0 ;/* fallthrough */ \ -case 72 : \ - _pp[8 ] =0 ;/* fallthrough */ \ -default : \ - _pp[7 ] =0 ; \ - _pp[6 ] =0 ; \ - _pp[5 ] =0 ; \ - _pp[4 ] =0 ; \ - _pp[3 ] =0 ; \ - _pp[2 ] =0 ; \ - _pp[1 ] =0 ; \ - _pp[0 ] =0 ; \ - } \ -}while (0 ) -/* PFNs are real physical page numbers. However, mem_map only begins to record * per-page information starting at pfn_base. This is to handle systems where * the first physical page in the machine is at some huge physical address,
include/linux/mm.h +34 -0
@@ -102,8 +102,42 @@ static inline void set_max_mapnr(unsigned long limit) { }* zeroing by defining this macro in <asm/pgtable.h>. */ #ifndef mm_zero_struct_page +#if BITS_PER_LONG = 64 +/* This function must be updated when the size of struct page grows above 80 +* or reduces below 64. The idea that compiler optimizes out switch() +* statement, and only leaves move/store instructions +*/ +#define mm_zero_struct_page (pp) __mm_zero_struct_page(pp) +static inline void __mm_zero_struct_page (struct page *page) +{ +unsigned long *_pp = (void *)page; + +/* Check that struct page is either 56, 64, 72, or 80 bytes */ +BUILD_BUG_ON (sizeof (struct page ) &7 ); +BUILD_BUG_ON (sizeof (struct page ) <56 ); +BUILD_BUG_ON (sizeof (struct page ) >80 ); + +switch (sizeof (struct page )) { +case 80 : + _pp[9 ] =0 ;/* fallthrough */ +case 72 : + _pp[8 ] =0 ;/* fallthrough */ +default : + _pp[7 ] =0 ;/* fallthrough */ +case 56 : + _pp[6 ] =0 ; + _pp[5 ] =0 ; + _pp[4 ] =0 ; + _pp[3 ] =0 ; + _pp[2 ] =0 ; + _pp[1 ] =0 ; + _pp[0 ] =0 ; + } +} +#else #define mm_zero_struct_page (pp) ((void)memset((pp), 0, sizeof(struct page)))#endif +#endif /* * Default maximum number of active map areas, this limits the number of vmas
Re: [mm PATCH v2 1/6] mm: Use mm_zero_struct_page from SPARC on all 64b architectures
I am worried about this change. I added SPARC optimized mm_zero_struct_page() specifically to SPARC because it has a poor performance with small memset()s, since it uses STBI instructions. However, other architectures might not suffer with small memset()s, and have hardware optimized memset variants for small sizes. Don't forget, this is a leaf routine on most arches, so the function call should be cheap. Also, the macro itself is not very flexible: when size of struct page is changed, it also must be modified (we could add fall throughs though), I would add this macro only to those arches that benefit from this change, in other words, I would like to see performance data.
I will review the rest of the patches in this series on Monday.
Thank you,
Pavel
Quoted 113 lines · Alexander Duyck
This change makes it so that we use the same approach that was already in use on Sparc on all the archtectures that support a 64b long.
This is mostly motivated by the fact that 8 to 10 store/move instructions are likely always going to be faster than having to call into a function that is not specialized for handling page init.
An added advantage to doing it this way is that the compiler can get away with combining writes in the
__init_single_pagecall. As a result the memset call will be reduced to only about 4 write operations, or at least that is what I am seeing with GCC 6.2 as the flags, LRU poitners, and count/mapcount seem to be cancelling out at least 4 of the 8 assignments on my system.One change I had to make to the function was to reduce the minimum page size to 56 to support some powerpc64 configurations.
Signed-off-by Alexander Duyck <[email protected]>--- arch/sparc/include/asm/pgtable_64.h | 30 ------------------------------ include/linux/mm.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-)arch/sparc/include/asm/pgtable_64.h +0
-30@@ -231,36 +231,6 @@extern struct page *mem_map_zero;#define ZERO_PAGE (vaddr) (mem_map_zero) -/* This macro must be updated when the size of struct page grows above 80 -* or reduces below 64. -* The idea that compiler optimizes out switch() statement, and only -* leaves clrx instructions -*/ -#define mm_zero_struct_page (pp) do { \ - unsigned long *_pp = (void *)(pp); \ - \ -/* Check that struct page is either 64, 72, or 80 bytes */ \ -BUILD_BUG_ON (sizeof (struct page ) &7 ); \ -BUILD_BUG_ON (sizeof (struct page ) <64 ); \ -BUILD_BUG_ON (sizeof (struct page ) >80 ); \ - \ -switch (sizeof (struct page )) { \ -case 80 : \ - _pp[9 ] =0 ;/* fallthrough */ \ -case 72 : \ - _pp[8 ] =0 ;/* fallthrough */ \ -default : \ - _pp[7 ] =0 ; \ - _pp[6 ] =0 ; \ - _pp[5 ] =0 ; \ - _pp[4 ] =0 ; \ - _pp[3 ] =0 ; \ - _pp[2 ] =0 ; \ - _pp[1 ] =0 ; \ - _pp[0 ] =0 ; \ - } \ -}while (0 ) -/* PFNs are real physical page numbers. However, mem_map only begins to record * per-page information starting at pfn_base. This is to handle systems where * the first physical page in the machine is at some huge physical address, include/linux/mm.h +34
-0@@ -102,8 +102,42 @@ static inline void set_max_mapnr(unsigned long limit) { }* zeroing by defining this macro in <asm/pgtable.h>. */ #ifndef mm_zero_struct_page +#if BITS_PER_LONG = 64 +/* This function must be updated when the size of struct page grows above 80 +* or reduces below 64. The idea that compiler optimizes out switch() +* statement, and only leaves move/store instructions +*/ +#define mm_zero_struct_page (pp) __mm_zero_struct_page(pp) +static inline void __mm_zero_struct_page (struct page *page) +{ +unsigned long *_pp = (void *)page; + +/* Check that struct page is either 56, 64, 72, or 80 bytes */ +BUILD_BUG_ON (sizeof (struct page ) &7 ); +BUILD_BUG_ON (sizeof (struct page ) <56 ); +BUILD_BUG_ON (sizeof (struct page ) >80 ); + +switch (sizeof (struct page )) { +case 80 : + _pp[9 ] =0 ;/* fallthrough */ +case 72 : + _pp[8 ] =0 ;/* fallthrough */ +default : + _pp[7 ] =0 ;/* fallthrough */ +case 56 : + _pp[6 ] =0 ; + _pp[5 ] =0 ; + _pp[4 ] =0 ; + _pp[3 ] =0 ; + _pp[2 ] =0 ; + _pp[1 ] =0 ; + _pp[0 ] =0 ; + } +} +#else #define mm_zero_struct_page (pp) ((void)memset((pp), 0, sizeof(struct page)))#endif +#endif /* * Default maximum number of active map areas, this limits the number of vmas
Re: [mm PATCH v2 1/6] mm: Use mm_zero_struct_page from SPARC on all 64b architectures
Well in the case of x86 the call to memset is expensive as well. In most cases it is 16 cycles plus 1 cycle per 16 bytes if I recall correctly. So for example in the case of skbuff which was a little over 192 bytes I know Jesper Brouer and myself were going back and forth with the idea of if we should try to do something similar.
I'm suspecting for the 64b architectures impacted by this change there should be little to no negative impact. The main reason for that being the fact that the compiler can actually drop some of the writes by merging them with the later assignments.
Thanks.
- Alex
Quoted 133 lines · Pavel Tatashin
I am worried about this change. I added SPARC optimized
mm_zero_struct_page()specifically to SPARC because it has a poor performance with small memset()s, since it uses STBI instructions. However, other architectures might not suffer with small memset()s, and have hardware optimized memset variants for small sizes. Don't forget, this is a leaf routine on most arches, so the function call should be cheap. Also, the macro itself is not very flexible: when size of struct page is changed, it also must be modified (we could add fall throughs though), I would add this macro only to those arches that benefit from this change, in other words, I would like to see performance data.I will review the rest of the patches in this series on Monday.
Thank you,
PavelQuoted 113 lines · Alexander Duyck
This change makes it so that we use the same approach that was already in use on Sparc on all the archtectures that support a 64b long.
This is mostly motivated by the fact that 8 to 10 store/move instructions are likely always going to be faster than having to call into a function that is not specialized for handling page init.
An added advantage to doing it this way is that the compiler can get away with combining writes in the
__init_single_pagecall. As a result the memset call will be reduced to only about 4 write operations, or at least that is what I am seeing with GCC 6.2 as the flags, LRU poitners, and count/mapcount seem to be cancelling out at least 4 of the 8 assignments on my system.One change I had to make to the function was to reduce the minimum page size to 56 to support some powerpc64 configurations.
Signed-off-by Alexander Duyck <[email protected]>--- arch/sparc/include/asm/pgtable_64.h | 30 ------------------------------ include/linux/mm.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-)arch/sparc/include/asm/pgtable_64.h +0
-30@@ -231,36 +231,6 @@extern struct page *mem_map_zero;#define ZERO_PAGE (vaddr) (mem_map_zero) -/* This macro must be updated when the size of struct page grows above 80 -* or reduces below 64. -* The idea that compiler optimizes out switch() statement, and only -* leaves clrx instructions -*/ -#define mm_zero_struct_page (pp) do { \ - unsigned long *_pp = (void *)(pp); \ - \ -/* Check that struct page is either 64, 72, or 80 bytes */ \ -BUILD_BUG_ON (sizeof (struct page ) &7 ); \ -BUILD_BUG_ON (sizeof (struct page ) <64 ); \ -BUILD_BUG_ON (sizeof (struct page ) >80 ); \ - \ -switch (sizeof (struct page )) { \ -case 80 : \ - _pp[9 ] =0 ;/* fallthrough */ \ -case 72 : \ - _pp[8 ] =0 ;/* fallthrough */ \ -default : \ - _pp[7 ] =0 ; \ - _pp[6 ] =0 ; \ - _pp[5 ] =0 ; \ - _pp[4 ] =0 ; \ - _pp[3 ] =0 ; \ - _pp[2 ] =0 ; \ - _pp[1 ] =0 ; \ - _pp[0 ] =0 ; \ - } \ -}while (0 ) -/* PFNs are real physical page numbers. However, mem_map only begins to record * per-page information starting at pfn_base. This is to handle systems where * the first physical page in the machine is at some huge physical address, include/linux/mm.h +34
-0@@ -102,8 +102,42 @@ static inline void set_max_mapnr(unsigned long limit) { }* zeroing by defining this macro in <asm/pgtable.h>. */ #ifndef mm_zero_struct_page +#if BITS_PER_LONG = 64 +/* This function must be updated when the size of struct page grows above 80 +* or reduces below 64. The idea that compiler optimizes out switch() +* statement, and only leaves move/store instructions +*/ +#define mm_zero_struct_page (pp) __mm_zero_struct_page(pp) +static inline void __mm_zero_struct_page (struct page *page) +{ +unsigned long *_pp = (void *)page; + +/* Check that struct page is either 56, 64, 72, or 80 bytes */ +BUILD_BUG_ON (sizeof (struct page ) &7 ); +BUILD_BUG_ON (sizeof (struct page ) <56 ); +BUILD_BUG_ON (sizeof (struct page ) >80 ); + +switch (sizeof (struct page )) { +case 80 : + _pp[9 ] =0 ;/* fallthrough */ +case 72 : + _pp[8 ] =0 ;/* fallthrough */ +default : + _pp[7 ] =0 ;/* fallthrough */ +case 56 : + _pp[6 ] =0 ; + _pp[5 ] =0 ; + _pp[4 ] =0 ; + _pp[3 ] =0 ; + _pp[2 ] =0 ; + _pp[1 ] =0 ; + _pp[0 ] =0 ; + } +} +#else #define mm_zero_struct_page (pp) ((void)memset((pp), 0, sizeof(struct page)))#endif +#endif /* * Default maximum number of active map areas, this limits the number of vmas
Re: [mm PATCH v2 1/6] mm: Use mm_zero_struct_page from SPARC on all 64b architectures
Still, lets get some real data, please provide Intel data before vs after. I could test on an ARM processor.
Pavel
Quoted 150 lines · Alexander Duyck
Well in the case of x86 the call to memset is expensive as well. In most cases it is 16 cycles plus 1 cycle per 16 bytes if I recall correctly. So for example in the case of skbuff which was a little over 192 bytes I know Jesper Brouer and myself were going back and forth with the idea of if we should try to do something similar.
I'm suspecting for the 64b architectures impacted by this change there should be little to no negative impact. The main reason for that being the fact that the compiler can actually drop some of the writes by merging them with the later assignments.
Thanks.
- Alex
Quoted 133 lines · Pavel Tatashin
I am worried about this change. I added SPARC optimized
mm_zero_struct_page()specifically to SPARC because it has a poor performance with small memset()s, since it uses STBI instructions. However, other architectures might not suffer with small memset()s, and have hardware optimized memset variants for small sizes. Don't forget, this is a leaf routine on most arches, so the function call should be cheap. Also, the macro itself is not very flexible: when size of struct page is changed, it also must be modified (we could add fall throughs though), I would add this macro only to those arches that benefit from this change, in other words, I would like to see performance data.I will review the rest of the patches in this series on Monday.
Thank you,
PavelQuoted 113 lines · Alexander Duyck
This change makes it so that we use the same approach that was already in use on Sparc on all the archtectures that support a 64b long.
This is mostly motivated by the fact that 8 to 10 store/move instructions are likely always going to be faster than having to call into a function that is not specialized for handling page init.
An added advantage to doing it this way is that the compiler can get away with combining writes in the
__init_single_pagecall. As a result the memset call will be reduced to only about 4 write operations, or at least that is what I am seeing with GCC 6.2 as the flags, LRU poitners, and count/mapcount seem to be cancelling out at least 4 of the 8 assignments on my system.One change I had to make to the function was to reduce the minimum page size to 56 to support some powerpc64 configurations.
Signed-off-by Alexander Duyck <[email protected]>--- arch/sparc/include/asm/pgtable_64.h | 30 ------------------------------ include/linux/mm.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-)arch/sparc/include/asm/pgtable_64.h +0
-30@@ -231,36 +231,6 @@extern struct page *mem_map_zero;#define ZERO_PAGE (vaddr) (mem_map_zero) -/* This macro must be updated when the size of struct page grows above 80 -* or reduces below 64. -* The idea that compiler optimizes out switch() statement, and only -* leaves clrx instructions -*/ -#define mm_zero_struct_page (pp) do { \ - unsigned long *_pp = (void *)(pp); \ - \ -/* Check that struct page is either 64, 72, or 80 bytes */ \ -BUILD_BUG_ON (sizeof (struct page ) &7 ); \ -BUILD_BUG_ON (sizeof (struct page ) <64 ); \ -BUILD_BUG_ON (sizeof (struct page ) >80 ); \ - \ -switch (sizeof (struct page )) { \ -case 80 : \ - _pp[9 ] =0 ;/* fallthrough */ \ -case 72 : \ - _pp[8 ] =0 ;/* fallthrough */ \ -default : \ - _pp[7 ] =0 ; \ - _pp[6 ] =0 ; \ - _pp[5 ] =0 ; \ - _pp[4 ] =0 ; \ - _pp[3 ] =0 ; \ - _pp[2 ] =0 ; \ - _pp[1 ] =0 ; \ - _pp[0 ] =0 ; \ - } \ -}while (0 ) -/* PFNs are real physical page numbers. However, mem_map only begins to record * per-page information starting at pfn_base. This is to handle systems where * the first physical page in the machine is at some huge physical address, include/linux/mm.h +34
-0@@ -102,8 +102,42 @@ static inline void set_max_mapnr(unsigned long limit) { }* zeroing by defining this macro in <asm/pgtable.h>. */ #ifndef mm_zero_struct_page +#if BITS_PER_LONG = 64 +/* This function must be updated when the size of struct page grows above 80 +* or reduces below 64. The idea that compiler optimizes out switch() +* statement, and only leaves move/store instructions +*/ +#define mm_zero_struct_page (pp) __mm_zero_struct_page(pp) +static inline void __mm_zero_struct_page (struct page *page) +{ +unsigned long *_pp = (void *)page; + +/* Check that struct page is either 56, 64, 72, or 80 bytes */ +BUILD_BUG_ON (sizeof (struct page ) &7 ); +BUILD_BUG_ON (sizeof (struct page ) <56 ); +BUILD_BUG_ON (sizeof (struct page ) >80 ); + +switch (sizeof (struct page )) { +case 80 : + _pp[9 ] =0 ;/* fallthrough */ +case 72 : + _pp[8 ] =0 ;/* fallthrough */ +default : + _pp[7 ] =0 ;/* fallthrough */ +case 56 : + _pp[6 ] =0 ; + _pp[5 ] =0 ; + _pp[4 ] =0 ; + _pp[3 ] =0 ; + _pp[2 ] =0 ; + _pp[1 ] =0 ; + _pp[0 ] =0 ; + } +} +#else #define mm_zero_struct_page (pp) ((void)memset((pp), 0, sizeof(struct page)))#endif +#endif /* * Default maximum number of active map areas, this limits the number of vmas
Re: [mm PATCH v2 1/6] mm: Use mm_zero_struct_page from SPARC on all 64b architectures
For the test system I had I was initializing 384GB of RAM per node, the average time per node dropped from 1.8s to 1.1s after this patch. In the case of the persistent memory on the system the initialization time for 3TB per node dropped from 24.8s to 23.8s on average. I saw similar results on my development system though I don't recall the exact amount, it was something like 640ms down to 470ms or so. Thanks.
- Alex
Quoted 158 lines · Pavel Tatashin
Still, lets get some real data, please provide Intel data before vs after. I could test on an ARM processor.
Pavel
Quoted 150 lines · Alexander Duyck
Well in the case of x86 the call to memset is expensive as well. In most cases it is 16 cycles plus 1 cycle per 16 bytes if I recall correctly. So for example in the case of skbuff which was a little over 192 bytes I know Jesper Brouer and myself were going back and forth with the idea of if we should try to do something similar.
I'm suspecting for the 64b architectures impacted by this change there should be little to no negative impact. The main reason for that being the fact that the compiler can actually drop some of the writes by merging them with the later assignments.
Thanks.
- Alex
Quoted 133 lines · Pavel Tatashin
I am worried about this change. I added SPARC optimized
mm_zero_struct_page()specifically to SPARC because it has a poor performance with small memset()s, since it uses STBI instructions. However, other architectures might not suffer with small memset()s, and have hardware optimized memset variants for small sizes. Don't forget, this is a leaf routine on most arches, so the function call should be cheap. Also, the macro itself is not very flexible: when size of struct page is changed, it also must be modified (we could add fall throughs though), I would add this macro only to those arches that benefit from this change, in other words, I would like to see performance data.I will review the rest of the patches in this series on Monday.
Thank you,
PavelQuoted 113 lines · Alexander Duyck
This change makes it so that we use the same approach that was already in use on Sparc on all the archtectures that support a 64b long.
This is mostly motivated by the fact that 8 to 10 store/move instructions are likely always going to be faster than having to call into a function that is not specialized for handling page init.
An added advantage to doing it this way is that the compiler can get away with combining writes in the
__init_single_pagecall. As a result the memset call will be reduced to only about 4 write operations, or at least that is what I am seeing with GCC 6.2 as the flags, LRU poitners, and count/mapcount seem to be cancelling out at least 4 of the 8 assignments on my system.One change I had to make to the function was to reduce the minimum page size to 56 to support some powerpc64 configurations.
Signed-off-by Alexander Duyck <[email protected]>--- arch/sparc/include/asm/pgtable_64.h | 30 ------------------------------ include/linux/mm.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 30 deletions(-)arch/sparc/include/asm/pgtable_64.h +0
-30@@ -231,36 +231,6 @@extern struct page *mem_map_zero;#define ZERO_PAGE (vaddr) (mem_map_zero) -/* This macro must be updated when the size of struct page grows above 80 -* or reduces below 64. -* The idea that compiler optimizes out switch() statement, and only -* leaves clrx instructions -*/ -#define mm_zero_struct_page (pp) do { \ - unsigned long *_pp = (void *)(pp); \ - \ -/* Check that struct page is either 64, 72, or 80 bytes */ \ -BUILD_BUG_ON (sizeof (struct page ) &7 ); \ -BUILD_BUG_ON (sizeof (struct page ) <64 ); \ -BUILD_BUG_ON (sizeof (struct page ) >80 ); \ - \ -switch (sizeof (struct page )) { \ -case 80 : \ - _pp[9 ] =0 ;/* fallthrough */ \ -case 72 : \ - _pp[8 ] =0 ;/* fallthrough */ \ -default : \ - _pp[7 ] =0 ; \ - _pp[6 ] =0 ; \ - _pp[5 ] =0 ; \ - _pp[4 ] =0 ; \ - _pp[3 ] =0 ; \ - _pp[2 ] =0 ; \ - _pp[1 ] =0 ; \ - _pp[0 ] =0 ; \ - } \ -}while (0 ) -/* PFNs are real physical page numbers. However, mem_map only begins to record * per-page information starting at pfn_base. This is to handle systems where * the first physical page in the machine is at some huge physical address, include/linux/mm.h +34
-0@@ -102,8 +102,42 @@ static inline void set_max_mapnr(unsigned long limit) { }* zeroing by defining this macro in <asm/pgtable.h>. */ #ifndef mm_zero_struct_page +#if BITS_PER_LONG = 64 +/* This function must be updated when the size of struct page grows above 80 +* or reduces below 64. The idea that compiler optimizes out switch() +* statement, and only leaves move/store instructions +*/ +#define mm_zero_struct_page (pp) __mm_zero_struct_page(pp) +static inline void __mm_zero_struct_page (struct page *page) +{ +unsigned long *_pp = (void *)page; + +/* Check that struct page is either 56, 64, 72, or 80 bytes */ +BUILD_BUG_ON (sizeof (struct page ) &7 ); +BUILD_BUG_ON (sizeof (struct page ) <56 ); +BUILD_BUG_ON (sizeof (struct page ) >80 ); + +switch (sizeof (struct page )) { +case 80 : + _pp[9 ] =0 ;/* fallthrough */ +case 72 : + _pp[8 ] =0 ;/* fallthrough */ +default : + _pp[7 ] =0 ;/* fallthrough */ +case 56 : + _pp[6 ] =0 ; + _pp[5 ] =0 ; + _pp[4 ] =0 ; + _pp[3 ] =0 ; + _pp[2 ] =0 ; + _pp[1 ] =0 ; + _pp[0 ] =0 ; + } +} +#else #define mm_zero_struct_page (pp) ((void)memset((pp), 0, sizeof(struct page)))#endif +#endif /* * Default maximum number of active map areas, this limits the number of vmas
PATCH v2 2/6 mm: Drop meminit_pfn_in_nid as it is redundant
In reply to Alexander DuyckAs best as I can tell the meminit_pfn_in_nid call is completely redundant. The deferred memory initialization is already making use of for_each_free_mem_range which in turn will call into __next_mem_range which will only return a memory range if it matches the node ID provided assuming it is not NUMA_NO_NODE.
I am operating on the assumption that there are no zones or pgdata_t structures that have a NUMA node of NUMA_NO_NODE associated with them. If that is the case then __next_mem_range will never return a memory range that doesn't match the zone's node ID and as such the check is redundant.
So one piece I would like to verfy on this is if this works for ia64. Technically it was using a different approach to get the node ID, but it seems to have the node ID also encoded into the memblock. So I am assuming this is okay, but would like to get confirmation on that.
--- mm/page_alloc.c | 50 ++++++++++++++------------------------------------ 1 file changed, 14 insertions(+), 36 deletions(-)
mm/page_alloc.c +14 -36
@@ -1305,36 +1305,22 @@ int __meminit early_pfn_to_nid(unsigned long pfn)#endif #ifdef CONFIG_NODES_SPAN_OTHER_NODES -static inline bool __meminit __maybe_unused -meminit_pfn_in_nid (unsigned long pfn,int node, -struct mminit_pfnnid_cache *state) +/* Only safe to use early in boot when initialisation is single-threaded */ +static inline bool __meminit early_pfn_in_nid (unsigned long pfn,int node) {int nid; - nid =__early_pfn_to_nid (pfn, state); + nid =__early_pfn_to_nid (pfn, &early_pfnnid_cache);if (nid >=0 && nid != node)return false;return true; } -/* Only safe to use early in boot when initialisation is single-threaded */ -static inline bool __meminit early_pfn_in_nid (unsigned long pfn,int node) -{ -return meminit_pfn_in_nid (pfn, node, &early_pfnnid_cache); -} -#else -static inline bool __meminit early_pfn_in_nid (unsigned long pfn,int node) {return true; } -static inline bool __meminit __maybe_unused -meminit_pfn_in_nid (unsigned long pfn,int node, -struct mminit_pfnnid_cache *state) -{ -return true; -}#endif
@@ -1463,21 +1449,13 @@ static inline void __init pgdat_init_report_one_done(void)* * Then, we check if a current large page is valid by only checking the validity * of the head pfn. -* -* Finally, meminit_pfn_in_nid is checked on systems where pfns can interleave -* within a node: a pfn is between start and end of a node, but does not belong -* to this memory node. */ -static inline bool __init -deferred_pfn_valid (int nid,unsigned long pfn, -struct mminit_pfnnid_cache *nid_init_state) +static inline bool __init deferred_pfn_valid (unsigned long pfn) {if (!pfn_valid_within (pfn))return false;if (!(pfn & (pageblock_nr_pages -1 )) && !pfn_valid (pfn))return false; -if (!meminit_pfn_in_nid (pfn, nid, nid_init_state)) -return false;return true; }
@@ -1485,15 +1463,14 @@ static inline void __init pgdat_init_report_one_done(void)* Free pages to buddy allocator. Try to free aligned pages in * pageblock_nr_pages sizes. */ -static void __init deferred_free_pages (int nid,int zid,unsigned long pfn, +static void __init deferred_free_pages (unsigned long pfn,unsigned long end_pfn) { -struct mminit_pfnnid_cache nid_init_state = { };unsigned long nr_pgmask = pageblock_nr_pages -1 ;unsigned long nr_free =0 ;for (; pfn < end_pfn; pfn++) { -if (!deferred_pfn_valid (nid, pfn, &nid_init_state)) { +if (!deferred_pfn_valid (pfn)) {deferred_free_range (pfn - nr_free, nr_free); nr_free =0 ; }else if (!(pfn & nr_pgmask)) {
@@ -1513,17 +1490,18 @@ static void __init deferred_free_pages(int nid, int zid, unsigned long pfn,* by performing it only once every pageblock_nr_pages. * Return number of pages initialized. */ -static unsigned long __init deferred_init_pages (int nid,int zid, +static unsigned long __init deferred_init_pages (struct zone *zone,unsigned long pfn,unsigned long end_pfn) { -struct mminit_pfnnid_cache nid_init_state = { };unsigned long nr_pgmask = pageblock_nr_pages -1 ; +int nid =zone_to_nid (zone);unsigned long nr_pages =0 ; +int zid =zone_idx (zone);struct page *page =NULL ;for (; pfn < end_pfn; pfn++) { -if (!deferred_pfn_valid (nid, pfn, &nid_init_state)) { +if (!deferred_pfn_valid (pfn)) { page =NULL ;continue ; }else if (!page || !(pfn & nr_pgmask)) {
@@ -1586,12 +1564,12 @@ static int __init deferred_init_memmap(void *data)for_each_free_mem_range (i, nid,MEMBLOCK_NONE , &spa, &epa,NULL ) { spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); - nr_pages +=deferred_init_pages (nid, zid, spfn, epfn); + nr_pages +=deferred_init_pages (zone, spfn, epfn); }for_each_free_mem_range (i, nid,MEMBLOCK_NONE , &spa, &epa,NULL ) { spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa)); -deferred_free_pages (nid, zid, spfn, epfn); +deferred_free_pages (spfn, epfn); }pgdat_resize_unlock (pgdat, &flags);
@@ -1680,7 +1658,7 @@ static int __init deferred_init_memmap(void *data)while (spfn < epfn && nr_pages < nr_pages_needed) { t =ALIGN (spfn +PAGES_PER_SECTION ,PAGES_PER_SECTION ); first_deferred_pfn =min (t, epfn); - nr_pages +=deferred_init_pages (nid, zid, spfn, + nr_pages +=deferred_init_pages (zone, spfn, first_deferred_pfn); spfn = first_deferred_pfn; }
@@ -1692,7 +1670,7 @@ static int __init deferred_init_memmap(void *data)for_each_free_mem_range (i, nid,MEMBLOCK_NONE , &spa, &epa,NULL ) { spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa)); epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa)); -deferred_free_pages (nid, zid, spfn, epfn); +deferred_free_pages (spfn, epfn);if (first_deferred_pfn = epfn)break ;
PATCH v2 3/6 mm: Use memblock/zone specific iterator for handling deferred page init
In reply to Alexander DuyckThis patch introduces a new iterator for_each_free_mem_pfn_range_in_zone.
This iterator will take care of making sure a given memory range provided is in fact contained within a zone. It takes are of all the bounds checking we were doing in deferred_grow_zone, and deferred_init_memmap. In addition it should help to speed up the search a bit by iterating until the end of a range is greater than the start of the zone pfn range, and will exit completely if the start is beyond the end of the zone.
This patch adds yet another iterator called for_each_free_mem_range_in_zone_from and then uses it to support initializing and freeing pages in groups no larger than MAX_ORDER_NR_PAGES. By doing this we can greatly improve the cache locality of the pages while we do several loops over them in the init and freeing process.
We are able to tighten the loops as a result since we only really need the checks for first_init_pfn in our first iteration and after that we can assume that all future values will be greater than this. So I have added a function called deferred_init_mem_pfn_range_in_zone that primes the iterators and if it fails we can just exit.
--- include/linux/memblock.h | 58 +++++++++++++++ mm/memblock.c | 63 ++++++++++++++++ mm/page_alloc.c | 176 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 242 insertions(+), 55 deletions(-)
include/linux/memblock.h +58 -0
@@ -178,6 +178,25 @@ void __next_reserved_mem_region(u64 *idx, phys_addr_t *out_start, p_start, p_end, p_nid))/** +* for_each_mem_range - iterate through memblock areas from type_a and not +* included in type_b. Or just type_a if type_b is NULL. +* @i: u64 used as loop variable +* @type_a: ptr to memblock_type to iterate +* @type_b: ptr to memblock_type which excludes from the iteration +* @nid: node selector, %NUMA_NO_NODE for all nodes +* @flags: pick from blocks based on memory attributes +* @p_start: ptr to phys_addr_t for start address of the range, can be %NULL +* @p_end: ptr to phys_addr_t for end address of the range, can be %NULL +* @p_nid: ptr to int for nid of the range, can be %NULL +*/ +#define for_each_mem_range_from (i, type_a, type_b, nid, flags, \ + p_start, p_end, p_nid) \ + for (i = 0, __next_mem_range(&i, nid, flags, type_a, type_b, \ + p_start, p_end, p_nid); \ + i != (u64)ULLONG_MAX; \ + __next_mem_range(&i, nid, flags, type_a, type_b, \ + p_start, p_end, p_nid)) +/** * for_each_mem_range_rev - reverse iterate through memblock areas from * type_a and not included in type_b. Or just type_a if type_b is NULL. * @i: u64 used as loop variable
@@ -248,6 +267,45 @@ void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, i >= 0; __next_mem_pfn_range(&i, nid, p_start, p_end, p_nid))#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +void __next_mem_pfn_range_in_zone (u64 *idx,struct zone *zone, +unsigned long *out_spfn, +unsigned long *out_epfn); +/** +* for_each_free_mem_range_in_zone - iterate through zone specific free +* memblock areas +* @i: u64 used as loop variable +* @zone: zone in which all of the memory blocks reside +* @p_start: ptr to phys_addr_t for start address of the range, can be %NULL +* @p_end: ptr to phys_addr_t for end address of the range, can be %NULL +* +* Walks over free (memory && !reserved) areas of memblock in a specific +* zone. Available as soon as memblock is initialized. +*/ +#define for_each_free_mem_pfn_range_in_zone (i, zone, p_start, p_end) \ + for (i = 0, \ + __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end); \ + i != (u64)ULLONG_MAX; \ + __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end)) + +/** +* for_each_free_mem_range_in_zone_from - iterate through zone specific +* free memblock areas from a given point +* @i: u64 used as loop variable +* @zone: zone in which all of the memory blocks reside +* @p_start: ptr to phys_addr_t for start address of the range, can be %NULL +* @p_end: ptr to phys_addr_t for end address of the range, can be %NULL +* +* Walks over free (memory && !reserved) areas of memblock in a specific +* zone, continuing from current position. Available as soon as memblock is +* initialized. +*/ +#define for_each_free_mem_pfn_range_in_zone_from (i, zone, p_start, p_end) \ + for (; i != (u64)ULLONG_MAX; \ + __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end)) + +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ +/** * for_each_free_mem_range - iterate through free memblock areas * @i: u64 used as loop variable
mm/memblock.c +63 -0
69 lines not on this page · raw message
mm/page_alloc.c +121 -55
222 lines not on this page · raw message
PATCH v2 4/6 mm: Do not set reserved flag for hotplug memory
In reply to Alexander DuyckThe general suspicion at this point is that the setting of the reserved bit is not really needed for hotplug memory. In addition the setting of this bit results in issues for DAX in that it is not possible to assign the region to KVM if the reserved bit is set in each page.
For now we can try just not setting the bit since we suspect it isn't adding value in setting it. If at a later time we find that it is needed we can come back through and re-add it for the hotplug paths.
--- mm/page_alloc.c | 11 ----------- 1 file changed, 11 deletions(-)
mm/page_alloc.c +0 -11
@@ -5571,8 +5571,6 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, page =pfn_to_page (pfn);__init_single_page (page, pfn, zone, nid); -if (context =MEMMAP_HOTPLUG ) -__SetPageReserved (page);/* * Mark the block movable so that blocks are reserved for
@@ -5626,15 +5624,6 @@ void __ref memmap_init_zone_device(struct zone *zone,__init_single_page (page, pfn, zone_idx, nid);/* -* Mark page reserved as it will need to wait for onlining -* phase for it to be fully associated with a zone. -* -* We can use the non-atomic __set_bit operation for setting -* the flag as we are still initializing the pages. -*/ -__SetPageReserved (page); - -/* * ZONE_DEVICE pages union ->lru with a ->pgmap back * pointer and hmm_data. It is a bug if a ZONE_DEVICE * page is ever freed or placed on a driver-private list.
Re: [mm PATCH v2 4/6] mm: Do not set reserved flag for hotplug memory
Quoted 22 lines · Alexander Duyck
The general suspicion at this point is that the setting of the reserved bit is not really needed for hotplug memory. In addition the setting of this bit results in issues for DAX in that it is not possible to assign the region to KVM if the reserved bit is set in each page.
For now we can try just not setting the bit since we suspect it isn't adding value in setting it. If at a later time we find that it is needed we can come back through and re-add it for the hotplug paths.
Suggested-by Michael Hocko <[email protected]>Reported-by Dan Williams <[email protected]>Signed-off-by Alexander Duyck <[email protected]>--- mm/page_alloc.c | 11 ----------- 1 file changed, 11 deletions(-)mm/page_alloc.c +0
-0
mm/page_alloc.c void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, · line 5575
Quoted 5 lines
mm/page_alloc.c +0
-2@@ -5571,8 +5571,6 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, page =pfn_to_page (pfn);__init_single_page (page, pfn, zone, nid); -if (context =MEMMAP_HOTPLUG ) -__SetPageReserved (page);
At a minimum I think we need to do this before removing PageReserved, to make sure zone_device pages are not tracked in the hibernation image.
kernel/power/snapshot.c +0 -0
@@ -1285,6 +1285,9 @@ static struct page *saveable_page(struct zone
*zone, unsigned long pfn)
if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
return NULL;+ if (is_zone_device_page(page))
+ return NULL;
+
if (PageReserved(page)
&& (!kernel_page_present(page) || pfn_is_nosave(pfn)))
return NULL;Re: [mm PATCH v2 4/6] mm: Do not set reserved flag for hotplug memory
Quoted 48 lines · Dan Williams
Quoted 22 lines · Alexander Duyck
The general suspicion at this point is that the setting of the reserved bit is not really needed for hotplug memory. In addition the setting of this bit results in issues for DAX in that it is not possible to assign the region to KVM if the reserved bit is set in each page.
For now we can try just not setting the bit since we suspect it isn't adding value in setting it. If at a later time we find that it is needed we can come back through and re-add it for the hotplug paths.
Suggested-by Michael Hocko <[email protected]>Reported-by Dan Williams <[email protected]>Signed-off-by Alexander Duyck <[email protected]>--- mm/page_alloc.c | 11 ----------- 1 file changed, 11 deletions(-)mm/page_alloc.c +0
-0mm/page_alloc.c void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, · line 5575
Quoted 5 lines
mm/page_alloc.c +0
-2@@ -5571,8 +5571,6 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, page =pfn_to_page (pfn);__init_single_page (page, pfn, zone, nid); -if (context =MEMMAP_HOTPLUG ) -__SetPageReserved (page);At a minimum I think we need to do this before removing PageReserved, to make sure
zone_devicepages are not tracked in the hibernation image.kernel/power/snapshot.c +3
-0@@ -1285,6 +1285,9 @@ static struct page *saveable_page(struct zone *zone, unsigned long pfn)if (swsusp_page_is_forbidden (page) ||swsusp_page_is_free (page))return NULL ; +if (is_zone_device_page (page)) +return NULL ; +if (PageReserved (page) && (!kernel_page_present (page) ||pfn_is_nosave (pfn)))
Quoted 2 lines
return NULL;
Yeah, I am pretty sure I broke hotplug in general with this too since it seems like it checks for the reserved flag before bringing a range online in online_pages_range.
I think I will drop this patch and go back to what I had before. There was a slight performance gain to be had for not setting the bit at all, but I think we are probably looking at yet another patch set if we want to go through and drop the need for the reserved bit to be set.
- Alex
PATCH v2 5/6 mm: Move hot-plug specific memory init into separate functions and optimize
In reply to Alexander DuyckThis patch is going through and combining the bits in memmap_init_zone and memmap_init_zone_device that are related to hotplug into a single function called __memmap_init_hotplug.
I also took the opportunity to integrate __init_single_page's functionality into this function. In doing so I can get rid of some of the redundancy such as the LRU pointers versus the pgmap.
--- mm/page_alloc.c | 199 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 137 insertions(+), 62 deletions(-)
mm/page_alloc.c +137 -62
@@ -1192,6 +1192,79 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,#endif } +static void __meminit __init_pageblock (unsigned long start_pfn, +unsigned long nr_pages, +unsigned long zone,int nid, +struct dev_pagemap *pgmap) +{ +unsigned long nr_pgmask = pageblock_nr_pages -1 ; +struct page *start_page =pfn_to_page (start_pfn); +unsigned long pfn = start_pfn + nr_pages -1 ; +#ifdef WANT_PAGE_VIRTUAL +bool is_highmem =is_highmem_idx (zone); +#endif +struct page *page; + +/* +* Enforce the following requirements: +* size > 0 +* size < pageblock_nr_pages +* start_pfn -> pfn does not cross pageblock_nr_pages boundary +*/ +VM_BUG_ON (((start_pfn ^ pfn) | (nr_pages -1 )) > nr_pgmask); + +/* +* Work from highest page to lowest, this way we will still be +* warm in the cache when we call set_pageblock_migratetype +* below. +* +* The loop is based around the page pointer as the main index +* instead of the pfn because pfn is not used inside the loop if +* the section number is not in page flags and WANT_PAGE_VIRTUAL +* is not defined. +*/ +for (page = start_page + nr_pages; page-- != start_page; pfn--) { +mm_zero_struct_page (page); +set_page_links (page, zone, nid, pfn); +init_page_count (page); +page_mapcount_reset (page); +page_cpupid_reset_last (page); + +/* +* ZONE_DEVICE pages union ->lru with a ->pgmap back +* pointer and hmm_data. It is a bug if a ZONE_DEVICE +* page is ever freed or placed on a driver-private list. +*/ +if (pgmap) + page->pgmap = pgmap; +else +INIT_LIST_HEAD (&page->lru); +#ifdef WANT_PAGE_VIRTUAL +/* The shift won't overflow because ZONE_NORMAL is below 4G. */ +if (!is_highmem) +set_page_address (page,__va (pfn <<PAGE_SHIFT )); +#endif + } + +/* +* Mark the block movable so that blocks are reserved for +* movable at startup. This will force kernel allocations +* to reserve their blocks rather than leaking throughout +* the address space during boot when many long-lived +* kernel allocations are made. +* +* bitmap is created for zone's valid pfn range. but memmap +* can be created for invalid pages (for alignment) +* check here not to call set_pageblock_migratetype() against +* pfn out of zone. +* +* Please note that MEMMAP_HOTPLUG path doesn't clear memmap +* because this is done early in sparse_add_one_section +*/ +if (!(start_pfn & nr_pgmask)) +set_pageblock_migratetype (start_page,MIGRATE_MOVABLE ); +} +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static void __meminit init_reserved_page (unsigned long pfn) {
@@ -5518,6 +5591,30 @@ void __ref build_all_zonelists(pg_data_t *pgdat)return false; } +static void __meminit __memmap_init_hotplug (unsigned long size,int nid, +unsigned long zone, +unsigned long start_pfn, +struct dev_pagemap *pgmap) +{ +unsigned long pfn = start_pfn + size; + +while (pfn != start_pfn) { +unsigned long stride = pfn; + + pfn =max (ALIGN_DOWN (pfn -1 , pageblock_nr_pages), start_pfn); + stride -= pfn; + +/* +* Mark page reserved as it will need to wait for +* onlining phase for it to be fully associated with +* a zone. +*/ +__init_pageblock (pfn, stride, zone, nid, pgmap); + +cond_resched (); + } +} +/* * Initially all pages are reserved - free ones are freed * up by memblock_free_all() once the early boot process is
@@ -5528,46 +5625,57 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, struct vmem_altmap *altmap) {unsigned long pfn, end_pfn = start_pfn + size; -struct page *page;if (highest_memmap_pfn < end_pfn -1 ) highest_memmap_pfn = end_pfn -1 ; +if (context =MEMMAP_HOTPLUG ) {#ifdef CONFIG_ZONE_DEVICE -/* -* Honor reservation requested by the driver for this ZONE_DEVICE -* memory. We limit the total number of pages to initialize to just -* those that might contain the memory mapping. We will defer the -* ZONE_DEVICE page initialization until after we have released -* the hotplug lock. -*/ -if (zone =ZONE_DEVICE ) { -if (!altmap) -return ; +/* +* Honor reservation requested by the driver for this +* ZONE_DEVICE memory. We limit the total number of pages to +* initialize to just those that might contain the memory +* mapping. We will defer the ZONE_DEVICE page initialization +* until after we have released the hotplug lock. +*/ +if (zone =ZONE_DEVICE ) { +if (!altmap) +return ; + +if (start_pfn = altmap->base_pfn) + start_pfn += altmap->reserve; + end_pfn = altmap->base_pfn + +vmem_altmap_offset (altmap); + } +#endif +/* +* For these pages we don't need to record the pgmap as they +* should represent only those pages used to store the memory +* map. The actual ZONE_DEVICE pages will be initialized later. +*/ +__memmap_init_hotplug (end_pfn - start_pfn, nid, zone, + start_pfn,NULL ); -if (start_pfn = altmap->base_pfn) - start_pfn += altmap->reserve; - end_pfn = altmap->base_pfn +vmem_altmap_offset (altmap); +return ; } -#endif for (pfn = start_pfn; pfn < end_pfn; pfn++) { +struct page *page; +/* * There can be holes in boot-time mem_map[]s handed to this * function. They do not exist on hotplugged memory. */ -if (context =MEMMAP_EARLY ) { -if (!early_pfn_valid (pfn)) { - pfn =next_valid_pfn (pfn) -1 ; -continue ; - } -if (!early_pfn_in_nid (pfn, nid)) -continue ; -if (overlap_memmap_init (zone, &pfn)) -continue ; -if (defer_init (nid, pfn, end_pfn)) -break ; +if (!early_pfn_valid (pfn)) { + pfn =next_valid_pfn (pfn) -1 ; +continue ; } +if (!early_pfn_in_nid (pfn, nid)) +continue ; +if (overlap_memmap_init (zone, &pfn)) +continue ; +if (defer_init (nid, pfn, end_pfn)) +break ; page =pfn_to_page (pfn);__init_single_page (page, pfn, zone, nid);
@@ -5597,14 +5705,12 @@ void __ref memmap_init_zone_device(struct zone *zone,unsigned long size, struct dev_pagemap *pgmap) { - unsigned long pfn, end_pfn = start_pfn + size; struct pglist_data *pgdat = zone->zone_pgdat;unsigned long zone_idx =zone_idx (zone);unsigned long start = jiffies;int nid = pgdat->node_id; -if (WARN_ON_ONCE (!pgmap || !is_dev_zone (zone))) -return ; +VM_BUG_ON (!is_dev_zone (zone));/* * The call to memmap_init_zone should have already taken care
@@ -5613,44 +5719,13 @@ void __ref memmap_init_zone_device(struct zone *zone,*/ if (pgmap->altmap_valid) {struct vmem_altmap *altmap = &pgmap->altmap; +unsigned long end_pfn = start_pfn + size; start_pfn = altmap->base_pfn +vmem_altmap_offset (altmap); size = end_pfn - start_pfn; } -for (pfn = start_pfn; pfn < end_pfn; pfn++) { -struct page *page =pfn_to_page (pfn); - -__init_single_page (page, pfn, zone_idx, nid); - -/* -* ZONE_DEVICE pages union ->lru with a ->pgmap back -* pointer and hmm_data. It is a bug if a ZONE_DEVICE -* page is ever freed or placed on a driver-private list. -*/ - page->pgmap = pgmap; - page->hmm_data =0 ; - -/* -* Mark the block movable so that blocks are reserved for -* movable at startup. This will force kernel allocations -* to reserve their blocks rather than leaking throughout -* the address space during boot when many long-lived -* kernel allocations are made. -* -* bitmap is created for zone's valid pfn range. but memmap -* can be created for invalid pages (for alignment) -* check here not to call set_pageblock_migratetype() against -* pfn out of zone. -* -* Please note that MEMMAP_HOTPLUG path doesn't clear memmap -* because this is done early in sparse_add_one_section -*/ -if (!(pfn & (pageblock_nr_pages -1 ))) { -set_pageblock_migratetype (page,MIGRATE_MOVABLE ); -cond_resched (); - } - } +__memmap_init_hotplug (size, nid, zone_idx, start_pfn, pgmap);pr_info ("%s initialised, %lu pages in %ums\n" ,dev_name (pgmap->dev), size,jiffies_to_msecs (jiffies - start));
PATCH v2 6/6 mm: Use common iterator for deferred_init_pages and deferred_free_pages
In reply to Alexander DuyckThis patch creates a common iterator to be used by both deferred_init_pages and deferred_free_pages. By doing this we can cut down a bit on code overhead as they will likely both be inlined into the same function anyway.
This new approach allows deferred_init_pages to make use of __init_pageblock. By doing this we can cut down on the code size by sharing code between both the hotplug and deferred memory init code paths.
An additional benefit to this approach is that we improve in cache locality of the memory init as we can focus on the memory areas related to identifying if a given PFN is valid and keep that warm in the cache until we transition to a region of a different type. So we will stream through a chunk of valid blocks before we turn to initializing page structs.
--- mm/page_alloc.c | 134 +++++++++++++++++++++++++++---------------------------- 1 file changed, 65 insertions(+), 69 deletions(-)
mm/page_alloc.c +65 -69
@@ -1477,32 +1477,6 @@ void clear_zone_contiguous(struct zone *zone) }#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -static void __init deferred_free_range (unsigned long pfn, -unsigned long nr_pages) -{ -struct page *page; -unsigned long i; - -if (!nr_pages) -return ; - - page =pfn_to_page (pfn); - -/* Free a large naturally-aligned chunk if possible */ -if (nr_pages = pageblock_nr_pages && - (pfn & (pageblock_nr_pages -1 )) =0 ) { -set_pageblock_migratetype (page,MIGRATE_MOVABLE ); -__free_pages_boot_core (page, pageblock_order); -return ; - } - -for (i =0 ; i < nr_pages; i++, page++, pfn++) { -if ((pfn & (pageblock_nr_pages -1 )) =0 ) -set_pageblock_migratetype (page,MIGRATE_MOVABLE ); -__free_pages_boot_core (page,0 ); - } -} -/* Completion tracking for deferred_init_memmap() threads */ static atomic_t pgdat_init_n_undone__initdata ;static __initdata DECLARE_COMPLETION (pgdat_init_all_done_comp);
@@ -1514,48 +1488,77 @@ static inline void __init pgdat_init_report_one_done(void) }/* -* Returns true if page needs to be initialized or freed to buddy allocator. +* Returns count if page range needs to be initialized or freed * -* First we check if pfn is valid on architectures where it is possible to have -* holes within pageblock_nr_pages. On systems where it is not possible, this -* function is optimized out. +* First, we check if a current large page is valid by only checking the +* validity of the head pfn. * -* Then, we check if a current large page is valid by only checking the validity -* of the head pfn. +* Then we check if the contiguous pfns are valid on architectures where it +* is possible to have holes within pageblock_nr_pages. On systems where it +* is not possible, this function is optimized out. */ -static inline bool __init deferred_pfn_valid (unsigned long pfn) +static unsigned long __next_pfn_valid_range (unsigned long *i, +unsigned long end_pfn) { -if (!pfn_valid_within (pfn)) -return false; -if (!(pfn & (pageblock_nr_pages -1 )) && !pfn_valid (pfn)) -return false; -return true; +unsigned long pfn = *i; +unsigned long count; + +while (pfn < end_pfn) { +unsigned long t =ALIGN (pfn +1 , pageblock_nr_pages); +unsigned long pageblock_pfn =min (t, end_pfn); + +#ifndef CONFIG_HOLES_IN_ZONE + count = pageblock_pfn - pfn; + pfn = pageblock_pfn; +if (!pfn_valid (pfn)) +continue ; +#else +for (count =0 ; pfn < pageblock_pfn; pfn++) { +if (pfn_valid_within (pfn)) { + count++; +continue ; + } + +if (count) +break ; + } + +if (!count) +continue ; +#endif + *i = pfn; +return count; + } + +return 0 ; } +#define for_each_deferred_pfn_valid_range (i, start_pfn, end_pfn, pfn, count) \ + for (i = (start_pfn), \ + count = __next_pfn_valid_range(&i, (end_pfn)); \ + count && ({ pfn = i - count; 1; }); \ + count = __next_pfn_valid_range(&i, (end_pfn)))/* * Free pages to buddy allocator. Try to free aligned pages in * pageblock_nr_pages sizes. */ -static void __init deferred_free_pages (unsigned long pfn, +static void __init deferred_free_pages (unsigned long start_pfn,unsigned long end_pfn) { -unsigned long nr_pgmask = pageblock_nr_pages -1 ; -unsigned long nr_free =0 ; - -for (; pfn < end_pfn; pfn++) { -if (!deferred_pfn_valid (pfn)) { -deferred_free_range (pfn - nr_free, nr_free); - nr_free =0 ; - }else if (!(pfn & nr_pgmask)) { -deferred_free_range (pfn - nr_free, nr_free); - nr_free =1 ; -touch_nmi_watchdog (); +unsigned long i, pfn, count; + +for_each_deferred_pfn_valid_range (i, start_pfn, end_pfn, pfn, count) { +struct page *page =pfn_to_page (pfn); + +if (count = pageblock_nr_pages) { +__free_pages_boot_core (page, pageblock_order); }else { - nr_free++; +while (count--) +__free_pages_boot_core (page++,0 ); } + +touch_nmi_watchdog (); } -/* Free the last block of pages to allocator */ -deferred_free_range (pfn - nr_free, nr_free); }/*
@@ -1564,29 +1567,22 @@ static void __init deferred_free_pages(unsigned long pfn,* Return number of pages initialized. */ static unsigned long __init deferred_init_pages (struct zone *zone, -unsigned long pfn, +unsigned long start_pfn,unsigned long end_pfn) { -unsigned long nr_pgmask = pageblock_nr_pages -1 ; +unsigned long i, pfn, count;int nid =zone_to_nid (zone);unsigned long nr_pages =0 ;int zid =zone_idx (zone); -struct page *page =NULL ; -for (; pfn < end_pfn; pfn++) { -if (!deferred_pfn_valid (pfn)) { - page =NULL ; -continue ; - }else if (!page || !(pfn & nr_pgmask)) { - page =pfn_to_page (pfn); -touch_nmi_watchdog (); - }else { - page++; - } -__init_single_page (page, pfn, zid, nid); - nr_pages++; +for_each_deferred_pfn_valid_range (i, start_pfn, end_pfn, pfn, count) { + nr_pages += count; +__init_pageblock (pfn, count, zid, nid,NULL ); + +touch_nmi_watchdog (); } -return (nr_pages); + +return nr_pages; }/*