mm: Don't init and clear page count when MEMINIT_EARLY
Subsystems (5)
v5 cover mm: Don't init and clear page count when MEMINIT_EARLY
__init_single_page would init page count and __free_pages_core would clear it. A lot of pages don't need to do this when in MEMINIT_EARLY context. It's unnecessary and time-consuming.
The 1st patch is allow optional initialization of page count and PG_reserved flag. It's in preparation for the 2nd patch, it didn't change anything.
The 2nd patch only init page count for the reserved region, not all of the region when MEMINIT_EARLY.
Yajun Deng (2):
mm: allow optional initialization of page count and PG_reserved flag
mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY
mm/hugetlb.c | 2 +- mm/internal.h | 8 +++++++- mm/mm_init.c | 40 ++++++++++++++++++++++++++-------------- mm/page_alloc.c | 20 ++++++++++++-------- 4 files changed, 46 insertions(+), 24 deletions(-)
Signature · 1 line
2.25.1
PATCH v5 1/2 mm: allow optional initialization of page count and PG_reserved flag
__init_single_page() unconditionally resets page count which is unnecessary for reserved pages.
To allow skipping page count initialization and marking a page reserved in one go add flags parameter to __init_single_page().
No functional changes.
---
v5: change the subject and commit.
v4: remove the changes of __init_zone_device_page().
v3: Introduce enum init_page_flags.
v2: Introduce INIT_PAGE_COUNT and INIT_PAGE_RESERVED.
v1: https://lore.kernel.org/all/[email protected]/
--- mm/hugetlb.c | 2 +- mm/internal.h | 8 +++++++- mm/mm_init.c | 24 +++++++++++++----------- 3 files changed, 21 insertions(+), 13 deletions(-)
mm/hugetlb.c +1 -1
@@ -3064,7 +3064,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {struct page *page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid); +__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT );prep_compound_tail ((struct page *)folio, pfn - head_pfn); ret =page_ref_freeze (page,1 );VM_BUG_ON (!ret);
mm/internal.h +7 -1
@@ -1209,8 +1209,14 @@ struct vma_prepare {struct vm_area_struct *remove2; }; +enum page_init_flags { +INIT_PAGE_COUNT = (1 <<0 ), +INIT_PAGE_RESERVED = (1 <<1 ), +}; +void __meminit __init_single_page (struct page *page,unsigned long pfn, -unsigned long zone,int nid); +unsigned long zone,int nid, +enum page_init_flags flags);/* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
mm/mm_init.c +13 -11
@@ -557,11 +557,11 @@ static void __init find_zone_movable_pfns_for_nodes(void) }void __meminit __init_single_page (struct page *page,unsigned long pfn, -unsigned long zone,int nid) +unsigned long zone,int nid, +enum page_init_flags flags) {mm_zero_struct_page (page);set_page_links (page, zone, nid, pfn); -init_page_count (page);page_mapcount_reset (page);page_cpupid_reset_last (page);page_kasan_tag_reset (page);
@@ -572,6 +572,10 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,if (!is_highmem_idx (zone))set_page_address (page,__va (pfn <<PAGE_SHIFT ));#endif +if (flags &INIT_PAGE_COUNT ) +init_page_count (page); +if (flags &INIT_PAGE_RESERVED ) +__SetPageReserved (page); }#ifdef CONFIG_NUMA
@@ -714,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)if (zone_spans_pfn (zone, pfn))break ; } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + __init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT); }#else static inline void pgdat_set_deferred_range (pg_data_t *pgdat) {}
@@ -821,8 +825,8 @@ static void __init init_unavailable_range(unsigned long spfn, pfn =pageblock_end_pfn (pfn) -1 ;continue ; } -__init_single_page (pfn_to_page (pfn), pfn, zone, node); -__SetPageReserved (pfn_to_page (pfn)); +__init_single_page (pfn_to_page (pfn), pfn, zone, node, +INIT_PAGE_COUNT |INIT_PAGE_RESERVED ); pgcnt++; }
@@ -884,7 +888,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid); +__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT );if (context ==MEMINIT_HOTPLUG )__SetPageReserved (page);
@@ -967,9 +971,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap) { - -__init_single_page (page ,pfn ,zone_idx ,nid ); -/* * Mark page reserved as it will need to wait for onlining * phase for it to be fully associated with a zone.
@@ -977,7 +978,8 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,* We can use the non-atomic __set_bit operation for setting * the flag as we are still initializing the pages. */ -__SetPageReserved (page); +__init_single_page (page, pfn, zone_idx, nid, +INIT_PAGE_COUNT |INIT_PAGE_RESERVED );/* * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
@@ -2058,7 +2060,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, }else { page++; } -__init_single_page (page, pfn, zid, nid); +__init_single_page (page, pfn, zid, nid,INIT_PAGE_COUNT ); nr_pages++; }return (nr_pages);
Signature · 1 line
2.25.1
PATCH v5 2/2 mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY
In reply to Yajun Dengmemmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.
Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.
At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.
The following data was tested on an x86 machine with 190GB of RAM.
before:free_low_memory_core_early() 341ms
after:free_low_memory_core_early() 285ms
---
v5: add flags in memmap_init_range.
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/
--- mm/mm_init.c | 20 +++++++++++++++----- mm/page_alloc.c | 20 ++++++++++++-------- 2 files changed, 27 insertions(+), 13 deletions(-)
mm/mm_init.c +15 -5
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)if (zone_spans_pfn (zone, pfn))break ; } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT); + __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0); }#else static inline void pgdat_set_deferred_range (pg_data_t *pgdat) {}
@@ -756,8 +756,11 @@ void __meminit reserve_bootmem_region(phys_addr_t start,init_reserved_page (start_pfn, nid); -/* Avoid false-positive PageTail() */ -INIT_LIST_HEAD (&page->lru); +/* +* We didn't init page count in memmap_init_range when +* MEMINIT_EARLY, so it must init page count here. +*/ +init_page_count (page);/* * no need for atomic set_bit because the struct
@@ -850,6 +853,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone struct vmem_altmap *altmap, int migratetype) {unsigned long pfn, end_pfn = start_pfn + size; +enum page_init_flags flags =0 ;struct page *page;if (highest_memmap_pfn < end_pfn -1 )
@@ -888,9 +892,15 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone } page =pfn_to_page (pfn); -__init_single_page (page, pfn, zone, nid,INIT_PAGE_COUNT ); + +/* If the context is MEMINIT_EARLY, we will init page count and +* mark page reserved in reserve_bootmem_region, the free region +* wouldn't have page count and we will check the pages count +* in __free_pages_core. +*/ if (context ==MEMINIT_HOTPLUG ) -__SetPageReserved (page); + flags =INIT_PAGE_COUNT |INIT_PAGE_RESERVED ; +__init_single_page (page, pfn, zone, nid, flags);/* * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +12 -8
@@ -1289,18 +1289,22 @@ void __free_pages_core(struct page *page, unsigned int order)unsigned int loop;/* -* When initializing the memmap, __init_single_page() sets the refcount -* of all pages to 1 ("allocated"/"not free"). We have to set the -* refcount of all involved pages to 0. +* When initializing the memmap, memmap_init_range sets the refcount +* of all pages to 1 ("reserved" and "free") in hotplug context. We +* have to set the refcount of all involved pages to 0. Otherwise, +* we don't do it, as reserve_bootmem_region only set the refcount on +* reserve region ("reserved") in early context. */ -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { -prefetchw (p +1 ); +if (page_count (page)) { +prefetchw (p); +for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { +prefetchw (p +1 ); +__ClearPageReserved (p); +set_page_count (p,0 ); + }__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );atomic_long_add (nr_pages, &page_zone (page)->managed_pages);
Signature · 1 line
2.25.1