mm: Don't set and reset page count in MEMINIT_EARLY
Subsystems (5)
v2 cover mm: Don't set and reset page count in MEMINIT_EARLY
__init_single_page would set page count and __free_pages_core would reset it. A lot of pages don't need to do this when in MEMINIT_EARLY context. It's unnecessary and time-consuming.
The 1st patch is pass page count and reserved to __init_single_page. It's in preparation for the 2nd patch, it didn't change anything.
The 2nd patch only set page count for the reserved region, not all of the region.
Yajun Deng (2):
mm: pass page count and reserved to __init_single_page
mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY
mm/hugetlb.c | 2 +- mm/internal.h | 8 +++++++- mm/mm_init.c | 45 ++++++++++++++++++++++++++++----------------- mm/page_alloc.c | 20 ++++++++++++-------- 4 files changed, 48 insertions(+), 27 deletions(-)
Signature · 1 line
2.25.1
PATCH v2 1/2 mm: pass page count and reserved to __init_single_page
When we init a single page, we need to mark this page reserved if it does. And some pages may not need to set page count, such as compound pages.
Introduce INIT_PAGE_COUNT and INIT_PAGE_RESERVED, let the caller decide if it needs to set page count and mark page reserved or not.
--- mm/hugetlb.c | 2 +- mm/internal.h | 8 +++++++- mm/mm_init.c | 31 +++++++++++++++++-------------- 3 files changed, 25 insertions(+), 16 deletions(-)
mm/hugetlb.c +1 -1
@@ -3196,7 +3196,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {struct page *page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid); +__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT );prep_compound_tail ((struct page *)folio, pfn - head_pfn); ret =page_ref_freeze (page,1 );VM_BUG_ON (!ret);
mm/internal.h +7 -1
@@ -1209,8 +1209,14 @@ struct vma_prepare {struct vm_area_struct *remove2; }; +enum { +INIT_PAGE_COUNT = (1 <<0 ), +INIT_PAGE_RESERVED = (1 <<1 ), +}; +void __meminit __init_single_page (struct page *page,unsigned long pfn, -unsigned long zone,int nid); +unsigned long zone,int nid, +unsigned int flags);/* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
mm/mm_init.c +17 -14
@@ -557,11 +557,11 @@ static void __init find_zone_movable_pfns_for_nodes(void) }void __meminit __init_single_page (struct page *page,unsigned long pfn, -unsigned long zone,int nid) +unsigned long zone,int nid, +unsigned int flags) {mm_zero_struct_page (page);set_page_links (page, zone, nid, pfn); -init_page_count (page);page_mapcount_reset (page);page_cpupid_reset_last (page);page_kasan_tag_reset (page);
@@ -572,6 +572,10 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,if (!is_highmem_idx (zone))set_page_address (page,__va (pfn <<PAGE_SHIFT ));#endif +if (flags &INIT_PAGE_COUNT ) +init_page_count (page); +if (flags &INIT_PAGE_RESERVED ) +__SetPageReserved (page); }#ifdef CONFIG_NUMA
@@ -714,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)if (zone_spans_pfn (zone, pfn))break ; } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + __init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT); }#else static inline void pgdat_set_deferred_range (pg_data_t *pgdat) {}
@@ -821,8 +825,8 @@ static void __init init_unavailable_range(unsigned long spfn, pfn =pageblock_end_pfn (pfn) -1 ;continue ; } -__init_single_page (pfn_to_page (pfn), pfn, zone, node); -__SetPageReserved (pfn_to_page (pfn)); +__init_single_page (pfn_to_page (pfn), pfn, zone, node, +INIT_PAGE_COUNT |INIT_PAGE_RESERVED ); pgcnt++; }
@@ -884,7 +888,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid); +__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT );if (context ==MEMINIT_HOTPLUG )__SetPageReserved (page);
@@ -965,11 +969,9 @@ static void __init memmap_init(void)#ifdef CONFIG_ZONE_DEVICE static void __ref __init_zone_device_page (struct page *page,unsigned long pfn,unsigned long zone_idx,int nid, -struct dev_pagemap *pgmap) +struct dev_pagemap *pgmap, +unsigned int flags) { - -__init_single_page (page, pfn, zone_idx, nid); -/* * Mark page reserved as it will need to wait for onlining * phase for it to be fully associated with a zone.
@@ -977,7 +979,7 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,* We can use the non-atomic __set_bit operation for setting * the flag as we are still initializing the pages. */ -__SetPageReserved (page); +__init_single_page (page, pfn, zone_idx, nid, flags |INIT_PAGE_RESERVED );/* * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
@@ -1041,7 +1043,7 @@ static void __ref memmap_init_compound(struct page *head,for (pfn = head_pfn +1 ; pfn < end_pfn; pfn++) {struct page *page =pfn_to_page (pfn); -__init_zone_device_page (page, pfn, zone_idx, nid, pgmap); +__init_zone_device_page (page, pfn, zone_idx, nid, pgmap,0 );prep_compound_tail (head, pfn - head_pfn);set_page_count (page,0 );
@@ -1084,7 +1086,8 @@ void __ref memmap_init_zone_device(struct zone *zone,for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {struct page *page =pfn_to_page (pfn); -__init_zone_device_page (page, pfn, zone_idx, nid, pgmap); +__init_zone_device_page (page, pfn, zone_idx, nid, pgmap, +INIT_PAGE_COUNT );if (pfns_per_compound ==1 )continue ;
@@ -2058,7 +2061,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, }else { page++; } -__init_single_page (page, pfn, zid, nid); +__init_single_page (page, pfn, zid, nid,INIT_PAGE_COUNT ); nr_pages++; }return (nr_pages);
Signature · 1 line
2.25.1
Andrew Morton
maintainerRe: [PATCH v2 1/2] mm: pass page count and reserved to __init_single_page
Quoted 25 lines · On Mon, 25 Sep 2023 15:21:49 +0800 Yajun Deng <[email protected]> wrote:
When we init a single page, we need to mark this page reserved if it does. And some pages may not need to set page count, such as compound pages.
Introduce
INIT_PAGE_COUNTandINIT_PAGE_RESERVED, let the caller decide if it needs to set page count and mark page reserved or not....
mm/hugetlb.c +1
-1@@ -3196,7 +3196,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {struct page *page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid); +__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT );prep_compound_tail ((struct page *)folio, pfn - head_pfn); ret =page_ref_freeze (page,1 );VM_BUG_ON (!ret);mm/internal.h +0
-0
mm/internal.h struct vma_prepare { · line 1215
Quoted 7 lines
mm/internal.h +4
-0@@ -1209,8 +1209,14 @@ struct vma_prepare {struct vm_area_struct *remove2; }; +enum { +INIT_PAGE_COUNT = (1 <<0 ), +INIT_PAGE_RESERVED = (1 <<1 ), +};
It would be neater to give this enum a name
mm/internal.h struct vma_prepare { · line 1219
Quoted 4 lines
mm/internal.h +2
-1void __meminit __init_single_page (struct page *page,unsigned long pfn, -unsigned long zone,int nid); +unsigned long zone,int nid, +unsigned int flags);
Then use this enum for `flags'.
PATCH v2 2/2 mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY
In reply to Yajun Dengmemmap_init_range() would set page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.
Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.
At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.
The following data was tested on an x86 machine with 190GB of RAM.
before:free_low_memory_core_early() 341ms
after:free_low_memory_core_early() 285ms
--- mm/mm_init.c | 18 +++++++++++++----- mm/page_alloc.c | 20 ++++++++++++-------- 2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +13 -5
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)if (zone_spans_pfn (zone, pfn))break ; } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT); + __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0); }#else static inline void pgdat_set_deferred_range (pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,init_reserved_page (start_pfn, nid); -/* Avoid false-positive PageTail() */ -INIT_LIST_HEAD (&page->lru); +/* Set page count for the reserve region */ +init_page_count (page);/* * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT ); -if (context ==MEMINIT_HOTPLUG ) + +/* If the context is MEMINIT_EARLY, we will set page count and +* mark page reserved in reserve_bootmem_region, the free region +* wouldn't have page count and we will check the pages count +* in __free_pages_core. +*/ +__init_single_page (page, pfn, zone, nid,0 ); +if (context ==MEMINIT_HOTPLUG ) { +init_page_count (page);__SetPageReserved (page); + }/* * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +12 -8
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)unsigned int loop;/* -* When initializing the memmap, __init_single_page() sets the refcount -* of all pages to 1 ("allocated"/"not free"). We have to set the -* refcount of all involved pages to 0. +* When initializing the memmap, memmap_init_range sets the refcount +* of all pages to 1 ("reserved" and "free") in hotplug context. We +* have to set the refcount of all involved pages to 0. Otherwise, +* we don't do it, as reserve_bootmem_region only set the refcount on +* reserve region ("reserved") in early context. */ -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { -prefetchw (p +1 ); +if (page_count (page)) { +prefetchw (p); +for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { +prefetchw (p +1 ); +__ClearPageReserved (p); +set_page_count (p,0 ); + }__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );atomic_long_add (nr_pages, &page_zone (page)->managed_pages);
Signature · 1 line
2.25.1