Skip to contentSkip to conversation

mm: Don't set and reset page count in MEMINIT_EARLY

PATCH In review Yajun Deng 3 participants 34 messages Sep 28, 2023 – Oct 17, 2023

Subsystems (5)
lore
Share & tools
lore

Yajun Deng

author
#0 lore raw

v4 cover mm: Don't set and reset page count in MEMINIT_EARLY

__init_single_page would set page count and __free_pages_core would reset it. A lot of pages don't need to do this when in MEMINIT_EARLY context. It's unnecessary and time-consuming.

The 1st patch is pass page count and reserved to __init_single_page. It's in preparation for the 2nd patch, it didn't change anything.

The 2nd patch only set page count for the reserved region, not all of the region.

Yajun Deng (2):
mm: pass page count and reserved to __init_single_page
mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

 mm/hugetlb.c    |  2 +-
 mm/internal.h   |  8 +++++++-
 mm/mm_init.c    | 38 ++++++++++++++++++++++++--------------
 mm/page_alloc.c | 20 ++++++++++++--------
 4 files changed, 44 insertions(+), 24 deletions(-)
Signature · 1 line
2.25.1
author
#1 lore raw

PATCH v4 1/2 mm: pass page count and reserved to __init_single_page

When we init a single page, we need to mark that page reserved when it is reserved. And some pages need to reset page count, such as compound pages.

Introduce enum init_page_flags, the caller will init page count and mark page reserved by passing INIT_PAGE_COUNT and INIT_PAGE_RESERVED.

---
v4: move the changes of __init_zone_device_page().
v3: Introduce enum init_page_flags.
v2: Introduce INIT_PAGE_COUNT and INIT_PAGE_RESERVED.
v1: https://lore.kernel.org/all/[email protected]/

---
 mm/hugetlb.c  |  2 +-
 mm/internal.h |  8 +++++++-
 mm/mm_init.c  | 24 +++++++++++++-----------
 3 files changed, 21 insertions(+), 13 deletions(-)
mm/hugetlb.c +1 -1
index a82dc37669b0..bb9c334a8392 100644
@@ -3196,7 +3196,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
 	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
 		struct page *page = pfn_to_page(pfn);
 
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
 		prep_compound_tail((struct page *)folio, pfn - head_pfn);
 		ret = page_ref_freeze(page, 1);
 		VM_BUG_ON(!ret);
mm/internal.h +7 -1
index d7916f1e9e98..449891ad7fdb 100644
@@ -1209,8 +1209,14 @@ struct vma_prepare {
 	struct vm_area_struct *remove2;
 };
 
+enum init_page_flags {
+	INIT_PAGE_COUNT    = (1 << 0),
+	INIT_PAGE_RESERVED = (1 << 1),
+};
+
 void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid);
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags);
 
 /* shrinker related functions */
 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
mm/mm_init.c +13 -11
index 06a72c223bce..9716c8a7ade9 100644
@@ -557,11 +557,11 @@ static void __init find_zone_movable_pfns_for_nodes(void)
 }
 
 void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid)
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags)
 {
 	mm_zero_struct_page(page);
 	set_page_links(page, zone, nid, pfn);
-	init_page_count(page);
 	page_mapcount_reset(page);
 	page_cpupid_reset_last(page);
 	page_kasan_tag_reset(page);
@@ -572,6 +572,10 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	if (!is_highmem_idx(zone))
 		set_page_address(page, __va(pfn << PAGE_SHIFT));
 #endif
+	if (flags & INIT_PAGE_COUNT)
+		init_page_count(page);
+	if (flags & INIT_PAGE_RESERVED)
+		__SetPageReserved(page);
 }
 
 #ifdef CONFIG_NUMA
@@ -714,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
 		if (zone_spans_pfn(zone, pfn))
 			break;
 	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
 }
 #else
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -821,8 +825,8 @@ static void __init init_unavailable_range(unsigned long spfn,
 			pfn = pageblock_end_pfn(pfn) - 1;
 			continue;
 		}
-		__init_single_page(pfn_to_page(pfn), pfn, zone, node);
-		__SetPageReserved(pfn_to_page(pfn));
+		__init_single_page(pfn_to_page(pfn), pfn, zone, node,
+				   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
 		pgcnt++;
 	}
 
@@ -884,7 +888,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 		}
 
 		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
 		if (context == MEMINIT_HOTPLUG)
 			__SetPageReserved(page);
 
@@ -967,9 +971,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 					  unsigned long zone_idx, int nid,
 					  struct dev_pagemap *pgmap)
 {
-
-	__init_single_page(page, pfn, zone_idx, nid);
-
 	/*
 	 * Mark page reserved as it will need to wait for onlining
 	 * phase for it to be fully associated with a zone.
@@ -977,7 +978,8 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	 * We can use the non-atomic __set_bit operation for setting
 	 * the flag as we are still initializing the pages.
 	 */
-	__SetPageReserved(page);
+	__init_single_page(page, pfn, zone_idx, nid,
+			   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
 
 	/*
 	 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
@@ -2058,7 +2060,7 @@ static unsigned long  __init deferred_init_pages(struct zone *zone,
 		} else {
 			page++;
 		}
-		__init_single_page(page, pfn, zid, nid);
+		__init_single_page(page, pfn, zid, nid, INIT_PAGE_COUNT);
 		nr_pages++;
 	}
 	return (nr_pages);
Signature · 1 line
2.25.1
maintainer
#2 lore raw

Re: [PATCH v4 1/2] mm: pass page count and reserved to __init_single_page

Quoted 1 line · Yajun Deng

Subject: mm: pass page count and reserved to __init_single_page

We add pass flags that tell __init_single_page() how to initialize page count and PG_Reserved, I think a better subject would be:

mm: allow optional initialization of page count and PG_reserved flag

Quoted 6 lines

When we init a single page, we need to mark that page reserved when it is reserved. And some pages need to reset page count, such as compound pages.

Introduce enum init_page_flags, the caller will init page count and mark page reserved by passing INIT_PAGE_COUNT and INIT_PAGE_RESERVED.

This does not really describe why the change is needed. How about

__init_single_page() unconditionally resets page count which is unnecessary for reserved pages.

To allow skipping page count initialization and marking a page reserved in one go add flags parameter to __init_single_page().

No functional changes.

Quoted 30 lines

---
v4: move the changes of __init_zone_device_page().
v3: Introduce enum init_page_flags.
v2: Introduce INIT_PAGE_COUNT and INIT_PAGE_RESERVED.
v1: https://lore.kernel.org/all/[email protected]/

---
 mm/hugetlb.c  |  2 +-
 mm/internal.h |  8 +++++++-
 mm/mm_init.c  | 24 +++++++++++++-----------
 3 files changed, 21 insertions(+), 13 deletions(-)
mm/hugetlb.c +1 -1
index a82dc37669b0..bb9c334a8392 100644
@@ -3196,7 +3196,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
 	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
 		struct page *page = pfn_to_page(pfn);
 
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
 		prep_compound_tail((struct page *)folio, pfn - head_pfn);
 		ret = page_ref_freeze(page, 1);
 		VM_BUG_ON(!ret);
mm/internal.h +0 -0
index d7916f1e9e98..449891ad7fdb 100644

mm/internal.h struct vma_prepare { · line 1212

Quoted 4 lines
mm/internal.h +1 -0
@@ -1209,8 +1209,14 @@ struct vma_prepare {
 	struct vm_area_struct *remove2;
 };
 
+enum init_page_flags {

enum page_init_flags please

Quoted 41 lines
mm/internal.h +6 -1
+	INIT_PAGE_COUNT    = (1 << 0),
+	INIT_PAGE_RESERVED = (1 << 1),
+};
+
 void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid);
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags);
 
 /* shrinker related functions */
 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
mm/mm_init.c +6 -2
index 06a72c223bce..9716c8a7ade9 100644
@@ -557,11 +557,11 @@ static void __init find_zone_movable_pfns_for_nodes(void)
 }
 
 void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid)
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags)
 {
 	mm_zero_struct_page(page);
 	set_page_links(page, zone, nid, pfn);
-	init_page_count(page);
 	page_mapcount_reset(page);
 	page_cpupid_reset_last(page);
 	page_kasan_tag_reset(page);
@@ -572,6 +572,10 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	if (!is_highmem_idx(zone))
 		set_page_address(page, __va(pfn << PAGE_SHIFT));
 #endif
+	if (flags & INIT_PAGE_COUNT)
+		init_page_count(page);
+	if (flags & INIT_PAGE_RESERVED)
+		__SetPageReserved(page);
 }
 
 #ifdef CONFIG_NUMA

mm/mm_init.c static void __meminit init_reserved_page(unsigned long pfn, int nid) · line 721

Quoted 5 lines
mm/mm_init.c +1 -1
@@ -714,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
 		if (zone_spans_pfn(zone, pfn))
 			break;
 	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);

There is __SetPageReserved call a few lines below, it can be folded here.

Quoted 55 lines
mm/mm_init.c +6 -8
 }
 #else
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -821,8 +825,8 @@ static void __init init_unavailable_range(unsigned long spfn,
 			pfn = pageblock_end_pfn(pfn) - 1;
 			continue;
 		}
-		__init_single_page(pfn_to_page(pfn), pfn, zone, node);
-		__SetPageReserved(pfn_to_page(pfn));
+		__init_single_page(pfn_to_page(pfn), pfn, zone, node,
+				   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
 		pgcnt++;
 	}
 
@@ -884,7 +888,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 		}
 
 		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
 		if (context == MEMINIT_HOTPLUG)
 			__SetPageReserved(page);
 
@@ -967,9 +971,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 					  unsigned long zone_idx, int nid,
 					  struct dev_pagemap *pgmap)
 {
-
-	__init_single_page(page, pfn, zone_idx, nid);
-
 	/*
 	 * Mark page reserved as it will need to wait for onlining
 	 * phase for it to be fully associated with a zone.
@@ -977,7 +978,8 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	 * We can use the non-atomic __set_bit operation for setting
 	 * the flag as we are still initializing the pages.
 	 */
-	__SetPageReserved(page);
+	__init_single_page(page, pfn, zone_idx, nid,
+			   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
 
 	/*
 	 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
@@ -2058,7 +2060,7 @@ static unsigned long  __init deferred_init_pages(struct zone *zone,
 		} else {
 			page++;
 		}
-		__init_single_page(page, pfn, zid, nid);
+		__init_single_page(page, pfn, zid, nid, INIT_PAGE_COUNT);
 		nr_pages++;
 	}
 	return (nr_pages);
Signature · 1 line
2.25.1
Signature · 2 lines
Sincerely yours,
Mike.
author
#3 lore raw

Re: [PATCH v4 1/2] mm: pass page count and reserved to __init_single_page

Quoted 6 lines · Mike Rapoport
Quoted 1 line · Yajun Deng

Subject: mm: pass page count and reserved to __init_single_page

We add pass flags that tell __init_single_page() how to initialize page count and PG_Reserved, I think a better subject would be:

mm: allow optional initialization of page count and PG_reserved flag

Okay.

Quoted 8 lines
Quoted 6 lines

When we init a single page, we need to mark that page reserved when it is reserved. And some pages need to reset page count, such as compound pages.

Introduce enum init_page_flags, the caller will init page count and mark page reserved by passing INIT_PAGE_COUNT and INIT_PAGE_RESERVED.

This does not really describe why the change is needed. How about

Quoted 7 lines

__init_single_page() unconditionally resets page count which is unnecessary for reserved pages.

To allow skipping page count initialization and marking a page reserved in one go add flags parameter to __init_single_page().

No functional changes.

Okay.

Quoted 33 lines
Quoted 33 lines

---
v4: move the changes of __init_zone_device_page().
v3: Introduce enum init_page_flags.
v2: Introduce INIT_PAGE_COUNT and INIT_PAGE_RESERVED.
v1: https://lore.kernel.org/all/[email protected]/

---
  mm/hugetlb.c  |  2 +-
  mm/internal.h |  8 +++++++-
  mm/mm_init.c  | 24 +++++++++++++-----------
  3 files changed, 21 insertions(+), 13 deletions(-)
mm/hugetlb.c +1 -1
index a82dc37669b0..bb9c334a8392 100644
@@ -3196,7 +3196,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
  	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
  		struct page *page = pfn_to_page(pfn);
 
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
  		prep_compound_tail((struct page *)folio, pfn - head_pfn);
  		ret = page_ref_freeze(page, 1);
  		VM_BUG_ON(!ret);
mm/internal.h +0 -0
index d7916f1e9e98..449891ad7fdb 100644
@@ -1209,8 +1209,14 @@ struct vma_prepare {
  	struct vm_area_struct *remove2;
  };
 
Quoted 2 lines

mm/internal.h struct vma_prepare { · line 1212

Quoted 1 line
mm/internal.h +1 -0
+enum init_page_flags {

enum page_init_flags please

Okay.

Quoted 39 lines
Quoted 39 lines
mm/internal.h +6 -1
+	INIT_PAGE_COUNT    = (1 << 0),
+	INIT_PAGE_RESERVED = (1 << 1),
+};
+
  void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid);
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags);
 
  /* shrinker related functions */
  unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
mm/mm_init.c +6 -2
index 06a72c223bce..9716c8a7ade9 100644
@@ -557,11 +557,11 @@ static void __init find_zone_movable_pfns_for_nodes(void)
  }
 
  void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid)
+				  unsigned long zone, int nid,
+				  enum init_page_flags flags)
  {
  	mm_zero_struct_page(page);
  	set_page_links(page, zone, nid, pfn);
-	init_page_count(page);
  	page_mapcount_reset(page);
  	page_cpupid_reset_last(page);
  	page_kasan_tag_reset(page);
@@ -572,6 +572,10 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
  	if (!is_highmem_idx(zone))
  		set_page_address(page, __va(pfn << PAGE_SHIFT));
  #endif
+	if (flags & INIT_PAGE_COUNT)
+		init_page_count(page);
+	if (flags & INIT_PAGE_RESERVED)
+		__SetPageReserved(page);
  }
 
Quoted 9 lines

mm/mm_init.c static void __meminit init_reserved_page(unsigned long pfn, int nid) · line 721

Quoted 7 lines
mm/mm_init.c +1 -1
  #ifdef CONFIG_NUMA
@@ -714,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
  		if (zone_spans_pfn(zone, pfn))
  			break;
  	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);

There is __SetPageReserved call a few lines below, it can be folded here.

No, There is a #ifdef in front of it. If so, I need to add __SetPageReserved to another init_reserved_page().

And there is an return before __init_single_page.
I will change INIT_PAGE_COUNT to 0 in next patch.

Quoted 55 lines
Quoted 55 lines
mm/mm_init.c +6 -8
  }
  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -821,8 +825,8 @@ static void __init init_unavailable_range(unsigned long spfn,
  			pfn = pageblock_end_pfn(pfn) - 1;
  			continue;
  		}
-		__init_single_page(pfn_to_page(pfn), pfn, zone, node);
-		__SetPageReserved(pfn_to_page(pfn));
+		__init_single_page(pfn_to_page(pfn), pfn, zone, node,
+				   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
  		pgcnt++;
  	}
 
@@ -884,7 +888,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
  		}
 
  		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid);
+		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
  		if (context == MEMINIT_HOTPLUG)
  			__SetPageReserved(page);
 
@@ -967,9 +971,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
  					  unsigned long zone_idx, int nid,
  					  struct dev_pagemap *pgmap)
  {
-
-	__init_single_page(page, pfn, zone_idx, nid);
-
  	/*
  	 * Mark page reserved as it will need to wait for onlining
  	 * phase for it to be fully associated with a zone.
@@ -977,7 +978,8 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
  	 * We can use the non-atomic __set_bit operation for setting
  	 * the flag as we are still initializing the pages.
  	 */
-	__SetPageReserved(page);
+	__init_single_page(page, pfn, zone_idx, nid,
+			   INIT_PAGE_COUNT | INIT_PAGE_RESERVED);
 
  	/*
  	 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
@@ -2058,7 +2060,7 @@ static unsigned long  __init deferred_init_pages(struct zone *zone,
  		} else {
  			page++;
  		}
-		__init_single_page(page, pfn, zid, nid);
+		__init_single_page(page, pfn, zid, nid, INIT_PAGE_COUNT);
  		nr_pages++;
  	}
  	return (nr_pages);
Signature · 1 line
2.25.1
author
#4 lore raw

PATCH v4 2/2 mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

In reply to Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
 mm/mm_init.c    | 18 +++++++++++++-----
 mm/page_alloc.c | 20 ++++++++++++--------
 2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +13 -5
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
 		if (zone_spans_pfn(zone, pfn))
 			break;
 	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
 }
 #else
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
 
 			init_reserved_page(start_pfn, nid);
 
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */
+			init_page_count(page);
 
 			/*
 			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 		}
 
 		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
 			__SetPageReserved(page);
+		}
 
 		/*
 		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +12 -8
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
 	unsigned int loop;
 
 	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
 	 */
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
 		__ClearPageReserved(p);
 		set_page_count(p, 0);
 	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
 
 	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
 
Signature · 1 line
2.25.1
maintainer
#5 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 44 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
 mm/mm_init.c    | 18 +++++++++++++-----
 mm/page_alloc.c | 20 ++++++++++++--------
 2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
 		if (zone_spans_pfn(zone, pfn))
 			break;
 	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
 }
 #else
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 759

Quoted 6 lines
mm/mm_init.c +1 -2
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
 
 			init_reserved_page(start_pfn, nid);
 
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 10 lines
mm/mm_init.c +1 -2
+			init_page_count(page);
 
 			/*
 			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 		}
 
 		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 10 lines
mm/mm_init.c +9 -0
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
 			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 11 lines
mm/mm_init.c +1 -0
+		}
 
 		/*
 		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -0
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
 	unsigned int loop;
 

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1293

Quoted 10 lines
mm/page_alloc.c +5 -3
 	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
 	 */

Again, why hotplug and early init should be different?

Quoted 21 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
 		__ClearPageReserved(p);
 		set_page_count(p, 0);
 	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
 
 	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
 
Signature · 1 line
2.25.1
Signature · 2 lines
Sincerely yours,
Mike.
author
#6 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 46 lines · Mike Rapoport
Quoted 45 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
  mm/mm_init.c    | 18 +++++++++++++-----
  mm/page_alloc.c | 20 ++++++++++++--------
  2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
  		if (zone_spans_pfn(zone, pfn))
  			break;
  	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
  }
  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
 
Quoted 6 lines

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 759

Quoted 5 lines
mm/mm_init.c +1 -2
  			init_reserved_page(start_pfn, nid);
 
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Okay.

Quoted 8 lines
Quoted 8 lines
mm/mm_init.c +1 -0
+			init_page_count(page);
 
  			/*
  			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
  		}
 
Quoted 16 lines

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 13 lines
mm/mm_init.c +9 -2
  		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
  			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

No, the following code would cost more time than the current code in memmap_init().

if (context == MEMINIT_HOTPLUG)

	__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT | INIT_PAGE_RESERVED);

else

	__init_single_page(page, pfn, zone, nid, 0);
Quoted 14 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 11 lines
mm/mm_init.c +1 -0
+		}
 
  		/*
  		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -0
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
  	unsigned int loop;
 
Quoted 11 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1293

Quoted 10 lines
mm/page_alloc.c +5 -3
  	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
  	 */

Again, why hotplug and early init should be different?

I will add a comment that describes it will save boot time.

Quoted 22 lines
Quoted 22 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
  		__ClearPageReserved(p);
  		set_page_count(p, 0);
  	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
 
  	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
 
Signature · 1 line
2.25.1
maintainer
#7 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 79 lines · Yajun Deng
Quoted 42 lines · Mike Rapoport
Quoted 41 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
  mm/mm_init.c    | 18 +++++++++++++-----
  mm/page_alloc.c | 20 ++++++++++++--------
  2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
  		if (zone_spans_pfn(zone, pfn))
  			break;
  	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
  }
Quoted 8 lines

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 757

Quoted 7 lines
mm/mm_init.c +1 -2
  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
  			init_reserved_page(start_pfn, nid);
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Okay.

Quoted 14 lines
Quoted 14 lines
mm/mm_init.c +6 -2
+			init_page_count(page);
  			/*
  			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
  		}
  		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
Quoted 8 lines

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 5 lines
mm/mm_init.c +4 -0
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
  			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

No, the following code would cost more time than the current code in memmap_init().

Quoted 6 lines

if (context == MEMINIT_HOTPLUG)

	__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT | INIT_PAGE_RESERVED);

else

	__init_single_page(page, pfn, zone, nid, 0);

Sorry if I wasn't clear. What I meant was to have something along these lines:

	enum page_init_flags flags = 0;

	if (context == MEMINIT_HOTPLUG)
		flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED;
	__init_single_page(page, pfn, zone, nid, flags);
Quoted 23 lines
Quoted 15 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 12 lines
mm/mm_init.c +1 -0
+		}
  		/*
  		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
  	unsigned int loop;
  	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1293

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
  	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
  		__ClearPageReserved(p);
  		set_page_count(p, 0);
  	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
  	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
Signature · 1 line
2.25.1
Signature · 2 lines
Sincerely yours,
Mike.
author
#8 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 82 lines · Mike Rapoport
Quoted 81 lines · Yajun Deng
Quoted 42 lines · Mike Rapoport
Quoted 41 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
   mm/mm_init.c    | 18 +++++++++++++-----
   mm/page_alloc.c | 20 ++++++++++++--------
   2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
   		if (zone_spans_pfn(zone, pfn))
   			break;
   	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
   }
Quoted 8 lines

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 757

Quoted 7 lines
mm/mm_init.c +1 -2
   #else
   static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
   			init_reserved_page(start_pfn, nid);
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Okay.

Quoted 13 lines
Quoted 13 lines
mm/mm_init.c +6 -2
+			init_page_count(page);
   			/*
   			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
   		}
   		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
Quoted 8 lines

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 5 lines
mm/mm_init.c +4 -0
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
   			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

No, the following code would cost more time than the current code in memmap_init().

if (context == MEMINIT_HOTPLUG)

	__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT | INIT_PAGE_RESERVED);

else

Quoted 9 lines
Quoted 1 line
	__init_single_page(page, pfn, zone, nid, 0);

Sorry if I wasn't clear. What I meant was to have something along these lines:

	enum page_init_flags flags = 0;

	if (context == MEMINIT_HOTPLUG)
		flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED;
	__init_single_page(page, pfn, zone, nid, flags);

Okay, I'll test the time consumed in memmap_init().

Quoted 23 lines
Quoted 23 lines
Quoted 23 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 9 lines
mm/mm_init.c +1 -0
+		}
   		/*
   		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -0
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
   	unsigned int loop;

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 10 lines
mm/page_alloc.c +5 -3
   	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
   	 */

Again, why hotplug and early init should be different?

Quoted 6 lines
Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
   		__ClearPageReserved(p);
   		set_page_count(p, 0);
   	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
   	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
Signature · 1 line
2.25.1
maintainer
#9 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 123 lines · Yajun Deng
Quoted 82 lines · Mike Rapoport
Quoted 81 lines · Yajun Deng
Quoted 42 lines · Mike Rapoport
Quoted 41 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
   mm/mm_init.c    | 18 +++++++++++++-----
   mm/page_alloc.c | 20 ++++++++++++--------
   2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
   		if (zone_spans_pfn(zone, pfn))
   			break;
   	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
   }
Quoted 8 lines

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 757

Quoted 7 lines
mm/mm_init.c +1 -2
   #else
   static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
   			init_reserved_page(start_pfn, nid);
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Okay.

Quoted 13 lines
Quoted 13 lines
mm/mm_init.c +6 -2
+			init_page_count(page);
   			/*
   			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
   		}
   		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
Quoted 8 lines

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 5 lines
mm/mm_init.c +4 -0
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
   			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

No, the following code would cost more time than the current code in memmap_init().

if (context == MEMINIT_HOTPLUG)

	__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT | INIT_PAGE_RESERVED);

else

Quoted 9 lines
Quoted 1 line
	__init_single_page(page, pfn, zone, nid, 0);

Sorry if I wasn't clear. What I meant was to have something along these lines:

	enum page_init_flags flags = 0;

	if (context == MEMINIT_HOTPLUG)
		flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED;
	__init_single_page(page, pfn, zone, nid, flags);

Okay, I'll test the time consumed in memmap_init().

Quoted 23 lines
Quoted 23 lines
Quoted 23 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 9 lines
mm/mm_init.c +1 -0
+		}
   		/*
   		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -0
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
   	unsigned int loop;

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 10 lines
mm/page_alloc.c +5 -3
   	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
   	 */

Again, why hotplug and early init should be different?

Quoted 5 lines
Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

The if (page_count(page)) is really non-obvious...

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
   		__ClearPageReserved(p);
   		set_page_count(p, 0);
   	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
   	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
Signature · 1 line
2.25.1
Signature · 2 lines
Sincerely yours,
Mike.
author
#10 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 122 lines · Mike Rapoport
Quoted 121 lines · Yajun Deng
Quoted 82 lines · Mike Rapoport
Quoted 81 lines · Yajun Deng
Quoted 42 lines · Mike Rapoport
Quoted 41 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
    mm/mm_init.c    | 18 +++++++++++++-----
    mm/page_alloc.c | 20 ++++++++++++--------
    2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
    		if (zone_spans_pfn(zone, pfn))
    			break;
    	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
    }
Quoted 8 lines

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 757

Quoted 7 lines
mm/mm_init.c +1 -2
    #else
    static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
    			init_reserved_page(start_pfn, nid);
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Okay.

Quoted 13 lines
Quoted 13 lines
mm/mm_init.c +6 -2
+			init_page_count(page);
    			/*
    			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
    		}
    		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
Quoted 8 lines

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 5 lines
mm/mm_init.c +4 -0
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
    			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

No, the following code would cost more time than the current code in memmap_init().

if (context == MEMINIT_HOTPLUG)

	__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT | INIT_PAGE_RESERVED);

else

Quoted 9 lines
Quoted 1 line
	__init_single_page(page, pfn, zone, nid, 0);

Sorry if I wasn't clear. What I meant was to have something along these lines:

	enum page_init_flags flags = 0;

	if (context == MEMINIT_HOTPLUG)
		flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED;
	__init_single_page(page, pfn, zone, nid, flags);

Okay, I'll test the time consumed in memmap_init().

Quoted 28 lines
Quoted 23 lines
Quoted 15 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

Quoted 12 lines
mm/mm_init.c +1 -0
+		}
    		/*
    		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
    	unsigned int loop;
    	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1293

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
    	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

The following data is the time consumed in online_pages_range().

start_pfn   nr_pages    before   after
0x680000    0x80000     4.315ms  4.461ms
0x700000    0x80000     4.468ms  4.440ms
0x780000    0x80000     4.295ms  3.835ms
0x800000    0x80000     3.928ms  4.377ms
0x880000    0x80000     4.321ms  4.378ms
0x900000    0x80000     4.448ms  3.964ms
0x980000    0x80000     4.000ms  4.381ms
0xa00000    0x80000     4.459ms  4.255ms
sum                     34.234ms  34.091ms

As we can see, it doesn't become slower with ' if (page_count(page))' when MEMINIT_HOTPLUG.

Quoted 21 lines

The if (page_count(page)) is really non-obvious...

Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
mm/page_alloc.c +7 -5
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
    		__ClearPageReserved(p);
    		set_page_count(p, 0);
    	}
-	__ClearPageReserved(p);
-	set_page_count(p, 0);
    	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
Signature · 1 line
2.25.1
maintainer
#11 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 35 lines · Yajun Deng
Quoted 24 lines · Mike Rapoport
Quoted 23 lines · Yajun Deng
Quoted 22 lines · Mike Rapoport
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
    	unsigned int loop;
    	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
    	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

Quoted 5 lines

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

Signature · 2 lines
Sincerely yours,
Mike.
#12 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 39 lines · Mike Rapoport
Quoted 38 lines · Yajun Deng
Quoted 24 lines · Mike Rapoport
Quoted 23 lines · Yajun Deng
Quoted 22 lines · Mike Rapoport
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
     	unsigned int loop;
     	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
     	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

Quoted 6 lines
Quoted 2 lines

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Signature · 3 lines
Cheers,

David / dhildenb
maintainer
#13 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 43 lines · David Hildenbrand
Quoted 42 lines · Mike Rapoport
Quoted 40 lines · Yajun Deng
Quoted 24 lines · Mike Rapoport
Quoted 23 lines · Yajun Deng
Quoted 22 lines · Mike Rapoport
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
     	unsigned int loop;
     	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
     	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

Quoted 7 lines
Quoted 3 lines

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't check throughfully other code paths.
Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Quoted 5 lines
Signature · 3 lines
Cheers,

David / dhildenb
Signature · 2 lines
Sincerely yours,
Mike.
#14 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 48 lines · Mike Rapoport
Quoted 47 lines · David Hildenbrand
Quoted 45 lines · Mike Rapoport
Quoted 35 lines · Yajun Deng
Quoted 24 lines · Mike Rapoport
Quoted 23 lines · Yajun Deng
Quoted 22 lines · Mike Rapoport
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
mm/page_alloc.c +0 -2
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
      	unsigned int loop;
      	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
Quoted 8 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1292

Quoted 7 lines
mm/page_alloc.c +5 -1
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
      	 */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

Quoted 5 lines

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

Quoted 8 lines
Quoted 3 lines

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't check throughfully other code paths.
Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Absolutely agreed.

Signature · 3 lines
Cheers,

David / dhildenb
author
#15 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 60 lines · David Hildenbrand
Quoted 59 lines · Mike Rapoport
Quoted 57 lines · David Hildenbrand
Quoted 46 lines · Mike Rapoport
Quoted 45 lines · Yajun Deng
Quoted 31 lines · Mike Rapoport
Quoted 30 lines · Yajun Deng
Quoted 29 lines · Mike Rapoport
Quoted 24 lines
Quoted 16 lines
Quoted 16 lines
mm/page_alloc.c +0 -1
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page 
 *page, unsigned int order)
           unsigned int loop;
           /*
-     * When initializing the memmap, __init_single_page() 

sets the refcount
-     * of all pages to 1 ("allocated"/"not free"). We have to set the
-     * refcount of all involved pages to 0.
+     * When initializing the memmap, memmap_init_range sets the refcount
+     * of all pages to 1 ("reserved" and "free") in hotplug

Quoted 8 lines
Quoted 7 lines

context. We
+     * have to set the refcount of all involved pages to 0. Otherwise,
+     * we don't do it, as reserve_bootmem_region only set the refcount on
+     * reserve region ("reserved") in early context. */

Again, why hotplug and early init should be different?

Quoted 1 line

I will add a comment that describes it will save boot time.

But why do we need initialize struct pages differently at boot time vs memory hotplug?
Is there a reason memory hotplug cannot have page count set to 0 just like for pages reserved at boot time?

Quoted 8 lines
Quoted 5 lines

This patch just save boot time in MEMINIT_EARLY. If someone finds out that it can save time in

MEMINIT_HOTPLUG, I think it can be done in another patch later. I just keeping it in the same.

But it's not the same. It becomes slower after your patch and the code that frees the pages for MEMINIT_EARLY and MEMINIT_HOTPLUG becomes non-uniform for no apparent reason.

__free_pages_core will also be called by others, such as:
deferred_free_range, do_collection and memblock_free_late.

Quoted 6 lines
Quoted 2 lines

We couldn't remove  'if (page_count(page))' even if we set page count to 0 when MEMINIT_HOTPLUG.

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Quoted 7 lines
Quoted 4 lines

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't check throughfully other code paths.
Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Absolutely agreed.

I already sent v5  a few days ago. Comments, please...

maintainer
#16 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 23 lines · Yajun Deng
Quoted 21 lines · David Hildenbrand
Quoted 19 lines · Mike Rapoport
Quoted 8 lines
Quoted 7 lines

That 'if' breaks the invariant that __free_pages_core is always called for
pages with initialized page count. Adding it may lead to subtle bugs and
random memory corruption so we don't want to add it at the first place.

Quoted 5 lines

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move
some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't
check throughfully other code paths.
Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Quoted 3 lines
Quoted 2 lines

Absolutely agreed.

I already sent v5  a few days ago. Comments, please...

Does it address all the feedback from this thread?

Signature · 2 lines
Sincerely yours,
Mike.
author
#17 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 19 lines · Mike Rapoport
Quoted 18 lines · Yajun Deng
Quoted 17 lines · David Hildenbrand
Quoted 16 lines · Mike Rapoport
Quoted 11 lines
Quoted 6 lines

That 'if' breaks the invariant that __free_pages_core is always called for
pages with initialized page count. Adding it may lead to subtle bugs and
random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move
some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't
check throughfully other code paths.
Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

Absolutely agreed.

I already sent v5  a few days ago. Comments, please...

Does it address all the feedback from this thread?

Except hotplug. As far as I konw, we only clear page count in MEMINIT_EARLY and all tail pages in compound page.

So adding 'if (page_count(page))' will have no actual effect for other case. According to previous data, it didn't

become slower in hotplug.

maintainer
#18 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 16 lines · Yajun Deng
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
Quoted 10 lines
Quoted 5 lines

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't check throughfully other code paths. Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

Absolutely agreed.

I already sent v5  a few days ago. Comments, please...

Does it address all the feedback from this thread?

Except hotplug.

Please reread carefully the last comments from me and from David above.

Signature · 2 lines
Sincerely yours,
Mike.
author
#19 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 13 lines · Mike Rapoport
Quoted 12 lines · Yajun Deng
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 8 lines
Quoted 4 lines

That 'if' breaks the invariant that __free_pages_core is always called for pages with initialized page count. Adding it may lead to subtle bugs and random memory corruption so we don't want to add it at the first place.

As long as we have to special-case memory hotplug, we know that we are always coming via generic_online_page() in that case. We could either move some logic over there, or let __free_pages_core() know what it should do.

Looks like the patch rather special cases MEMINIT_EARLY, although I didn't check throughfully other code paths. Anyway, relying on page_count() to be correct in different ways for different callers of __free_pages_core() does not sound right to me.

Quoted 7 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines
Quoted 2 lines

Absolutely agreed.

I already sent v5  a few days ago. Comments, please...

Does it address all the feedback from this thread?

Except hotplug.

Please reread carefully the last comments from me and from David above.

I replied in another thread about that 'if' statement. David just replied to me, let's discuss in another thread.

#20 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

In reply to Mike Rapoport
Quoted 79 lines · Mike Rapoport
Quoted 44 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early() 341ms

after:
free_low_memory_core_early() 285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/

---
  mm/mm_init.c    | 18 +++++++++++++-----
  mm/page_alloc.c | 20 ++++++++++++--------
  2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit init_reserved_page(unsigned long pfn, int nid)
  		if (zone_spans_pfn(zone, pfn))
  			break;
  	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, INIT_PAGE_COUNT);
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
  }
  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c void __meminit reserve_bootmem_region(phys_addr_t start, · line 759

Quoted 6 lines
mm/mm_init.c +1 -2
@@ -756,8 +756,8 @@ void __meminit reserve_bootmem_region(phys_addr_t start,
 
  			init_reserved_page(start_pfn, nid);
 
-			/* Avoid false-positive PageTail() */
-			INIT_LIST_HEAD(&page->lru);
+			/* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 10 lines
mm/mm_init.c +1 -2
+			init_page_count(page);
 
  			/*
  			 * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
  		}
 
  		page = pfn_to_page(pfn);
-		__init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT);
-		if (context == MEMINIT_HOTPLUG)

mm/mm_init.c void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone · line 900

Quoted 10 lines
mm/mm_init.c +9 -0
+
+		/* If the context is MEMINIT_EARLY, we will init page count and
+		 * mark page reserved in reserve_bootmem_region, the free region
+		 * wouldn't have page count and we will check the pages count
+		 * in __free_pages_core.
+		 */
+		__init_single_page(page, pfn, zone, nid, 0);
+		if (context == MEMINIT_HOTPLUG) {
+			init_page_count(page);
  			__SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

Quoted 2 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

There are a lot of details to that, and I'll share some I can briefly think of.

1) __SetPageReserved()

I tried removing that a while ago, but there was a blocker (IIRC something about ZONE_DEVICE). I still have the patches at [1] and I could probably take a look if that blocker still exists (I recall that something changed at some point, but I never had the time to follow up).

But once we stop setting the pages reserved, we might run into issues with ...

2) init_page_count()

virtio-mem, XEN balloon and HV-balloon add memory blocks that can contain holes. set_online_page_callback() is used to intercept memory onlining and to expose only the pages that are not holes to the buddy: calling generic_online_page() on !hole.

Holes are PageReserved but with an initialized page count. Memory offlining will fail on PageReserved pages -- has_unmovable_pages().

At least virtio-mem clears the PageReserved flag of holes when onlining memory, and currently relies in the page count to be reasonable (so memory offlining can work).

static void virtio_mem_set_fake_offline(unsigned long pfn,
unsigned long nr_pages, bool onlined)

{
	page_offline_begin();
	for (; nr_pages--; pfn++) {
		struct page *page = pfn_to_page(pfn);

		__SetPageOffline(page);
		if (!onlined) {
			SetPageDirty(page);
			/* FIXME: remove after cleanups */
			ClearPageReserved(page);
		}
	}
	page_offline_end();
}

For virtio-mem, we could initialize the page count there instead. The other PV drivers might require a bit more thought.

[1] https://github.com/davidhildenbrand/linux/tree/online_reserved_cleanup

Quoted 25 lines
Quoted 12 lines
mm/mm_init.c +1 -0
+		}
 
  		/*
  		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
mm/page_alloc.c +0 -0
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, unsigned int order)
  	unsigned int loop;
 

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1293

Quoted 10 lines
mm/page_alloc.c +5 -3
  	/*
-	 * When initializing the memmap, __init_single_page() sets the refcount
-	 * of all pages to 1 ("allocated"/"not free"). We have to set the
-	 * refcount of all involved pages to 0.
+	 * When initializing the memmap, memmap_init_range sets the refcount
+	 * of all pages to 1 ("reserved" and "free") in hotplug context. We
+	 * have to set the refcount of all involved pages to 0. Otherwise,
+	 * we don't do it, as reserve_bootmem_region only set the refcount on
+	 * reserve region ("reserved") in early context.
  	 */

Again, why hotplug and early init should be different?

Quoted 12 lines

mm/page_alloc.c void __free_pages_core(struct page *page, unsigned int order) · line 1302

Quoted 12 lines
mm/page_alloc.c +7 -3
-	prefetchw(p);
-	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
-		prefetchw(p + 1);
+	if (page_count(page)) {
+		prefetchw(p);
+		for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
+			prefetchw(p + 1);
+			__ClearPageReserved(p);
+			set_page_count(p, 0);
+		}
  		__ClearPageReserved(p);
  		set_page_count(p, 0);

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Signature · 3 lines
Cheers,

David / dhildenb
author
#21 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 175 lines · David Hildenbrand
Quoted 81 lines · Mike Rapoport
Quoted 48 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early()    341ms

after:
free_low_memory_core_early()    285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/
---
  mm/mm_init.c    | 18 +++++++++++++-----
  mm/page_alloc.c | 20 ++++++++++++--------

  2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit 
 init_reserved_page(unsigned long pfn, int nid)
           if (zone_spans_pfn(zone, pfn))
               break;
       }
-    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 
 INIT_PAGE_COUNT);
+    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
   }

  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c +0 -0
@@ -756,8 +756,8 @@ void __meminit 
 reserve_bootmem_region(phys_addr_t start,
                 init_reserved_page(start_pfn, nid);
Quoted 3 lines

  -            /* Avoid false-positive PageTail() */ -            INIT_LIST_HEAD(&page->lru);
+            /* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 13 lines

+            init_page_count(page);
                /*
               * no need for atomic set_bit because the struct

mm/mm_init.c +0 -0
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long 

size, int nid, unsigned long zone
          }
            page = pfn_to_page(pfn);
-        __init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT); -        if (context == MEMINIT_HOTPLUG)
+
+        /* If the context is MEMINIT_EARLY, we will init page count and
+         * mark page reserved in reserve_bootmem_region, the free

Quoted 8 lines

region
+         * wouldn't have page count and we will check the pages count +         * in __free_pages_core.
+         */
+        __init_single_page(page, pfn, zone, nid, 0);
+        if (context == MEMINIT_HOTPLUG) {
+            init_page_count(page); __SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

Quoted 2 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

There are a lot of details to that, and I'll share some I can briefly think of.

1) __SetPageReserved()

I tried removing that a while ago, but there was a blocker (IIRC something about ZONE_DEVICE). I still have the patches at [1] and I could probably take a look if that blocker still exists (I recall that something changed at some point, but I never had the time to follow up).

But once we stop setting the pages reserved, we might run into issues with ...

2) init_page_count()

virtio-mem, XEN balloon and HV-balloon add memory blocks that can contain holes. set_online_page_callback() is used to intercept memory onlining and to expose only the pages that are not holes to the buddy: calling generic_online_page() on !hole.

Holes are PageReserved but with an initialized page count. Memory offlining will fail on PageReserved pages -- has_unmovable_pages().

At least virtio-mem clears the PageReserved flag of holes when onlining memory, and currently relies in the page count to be reasonable (so memory offlining can work).

static void virtio_mem_set_fake_offline(unsigned long pfn, unsigned long nr_pages, bool onlined) {
    page_offline_begin();
    for (; nr_pages--; pfn++) {
        struct page *page = pfn_to_page(pfn);

        __SetPageOffline(page);
        if (!onlined) {
            SetPageDirty(page);
            /* FIXME: remove after cleanups */
            ClearPageReserved(page);
        }
    }
    page_offline_end();
}

For virtio-mem, we could initialize the page count there instead. The other PV drivers might require a bit more thought.

[1] https://github.com/davidhildenbrand/linux/tree/online_reserved_cleanup

Quoted 19 lines
Quoted 19 lines

+        }
            /*
           * Usually, we want to mark the pageblock MIGRATE_MOVABLE,

mm/page_alloc.c +0 -1
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, 
 unsigned int order)
       unsigned int loop;
         /*
-     * When initializing the memmap, __init_single_page() sets the 

refcount
-     * of all pages to 1 ("allocated"/"not free"). We have to set the -     * refcount of all involved pages to 0.
+     * When initializing the memmap, memmap_init_range sets the refcount
+     * of all pages to 1 ("reserved" and "free") in hotplug

Quoted 21 lines
Quoted 6 lines

context. We
+     * have to set the refcount of all involved pages to 0. Otherwise, +     * we don't do it, as reserve_bootmem_region only set the refcount on
+     * reserve region ("reserved") in early context.
       */

Again, why hotplug and early init should be different?

Quoted 12 lines

-    prefetchw(p);
-    for (loop = 0; loop < (nr_pages - 1); loop++, p++) { -        prefetchw(p + 1);
+    if (page_count(page)) {
+        prefetchw(p);
+        for (loop = 0; loop < (nr_pages - 1); loop++, p++) { +            prefetchw(p + 1);
+            __ClearPageReserved(p);
+            set_page_count(p, 0);
+        }
          __ClearPageReserved(p);
          set_page_count(p, 0);

Quoted 4 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

author
#22 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 178 lines · Yajun Deng
Quoted 176 lines · David Hildenbrand
Quoted 81 lines · Mike Rapoport
Quoted 48 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context, and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early()    341ms

after:
free_low_memory_core_early()    285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it.
v1: https://lore.kernel.org/all/[email protected]/
---
  mm/mm_init.c    | 18 +++++++++++++-----
  mm/page_alloc.c | 20 ++++++++++++--------

  2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit 
 init_reserved_page(unsigned long pfn, int nid)
           if (zone_spans_pfn(zone, pfn))
               break;
       }
-    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 
 INIT_PAGE_COUNT);
+    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
   }

  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c +0 -0
@@ -756,8 +756,8 @@ void __meminit 
 reserve_bootmem_region(phys_addr_t start,
                 init_reserved_page(start_pfn, nid);
Quoted 3 lines

  -            /* Avoid false-positive PageTail() */ -            INIT_LIST_HEAD(&page->lru);
+            /* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 13 lines

+            init_page_count(page);
                /*
               * no need for atomic set_bit because the struct

mm/mm_init.c +0 -0
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long 

size, int nid, unsigned long zone
          }
            page = pfn_to_page(pfn);
-        __init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT); -        if (context == MEMINIT_HOTPLUG)
+
+        /* If the context is MEMINIT_EARLY, we will init page count and
+         * mark page reserved in reserve_bootmem_region, the free

Quoted 8 lines

region
+         * wouldn't have page count and we will check the pages count +         * in __free_pages_core.
+         */
+        __init_single_page(page, pfn, zone, nid, 0);
+        if (context == MEMINIT_HOTPLUG) {
+            init_page_count(page); __SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED an call __init_single_page() after the check for MEMINIT_HOTPLUG.

Quoted 2 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all. @David, can you comment please?

There are a lot of details to that, and I'll share some I can briefly think of.

1) __SetPageReserved()

I tried removing that a while ago, but there was a blocker (IIRC something about ZONE_DEVICE). I still have the patches at [1] and I could probably take a look if that blocker still exists (I recall that something changed at some point, but I never had the time to follow up).

But once we stop setting the pages reserved, we might run into issues with ...

2) init_page_count()

virtio-mem, XEN balloon and HV-balloon add memory blocks that can contain holes. set_online_page_callback() is used to intercept memory onlining and to expose only the pages that are not holes to the buddy: calling generic_online_page() on !hole.

Holes are PageReserved but with an initialized page count. Memory offlining will fail on PageReserved pages -- has_unmovable_pages().

At least virtio-mem clears the PageReserved flag of holes when onlining memory, and currently relies in the page count to be reasonable (so memory offlining can work).

static void virtio_mem_set_fake_offline(unsigned long pfn, unsigned long nr_pages, bool onlined) {
    page_offline_begin();
    for (; nr_pages--; pfn++) {
        struct page *page = pfn_to_page(pfn);

        __SetPageOffline(page);
        if (!onlined) {
            SetPageDirty(page);
            /* FIXME: remove after cleanups */
            ClearPageReserved(page);
        }
    }
    page_offline_end();
}

For virtio-mem, we could initialize the page count there instead. The other PV drivers might require a bit more thought.

[1] https://github.com/davidhildenbrand/linux/tree/online_reserved_cleanup

Quoted 41 lines
Quoted 18 lines

+        }
            /*
           * Usually, we want to mark the pageblock MIGRATE_MOVABLE,

mm/page_alloc.c +0 -1
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page, 
 unsigned int order)
       unsigned int loop;
         /*
-     * When initializing the memmap, __init_single_page() sets the 

refcount
-     * of all pages to 1 ("allocated"/"not free"). We have to set the -     * refcount of all involved pages to 0.
+     * When initializing the memmap, memmap_init_range sets the refcount

Quoted 8 lines

+     * of all pages to 1 ("reserved" and "free") in hotplug context. We
+     * have to set the refcount of all involved pages to 0. Otherwise,
+     * we don't do it, as reserve_bootmem_region only set the refcount on
+     * reserve region ("reserved") in early context. */

Again, why hotplug and early init should be different?

Quoted 12 lines

-    prefetchw(p);
-    for (loop = 0; loop < (nr_pages - 1); loop++, p++) { -        prefetchw(p + 1);
+    if (page_count(page)) {
+        prefetchw(p);
+        for (loop = 0; loop < (nr_pages - 1); loop++, p++) { +            prefetchw(p + 1);
+            __ClearPageReserved(p);
+            set_page_count(p, 0);
+        }
          __ClearPageReserved(p);
          set_page_count(p, 0);

Quoted 8 lines
Quoted 4 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

	if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {
#23 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 210 lines · Yajun Deng
Quoted 201 lines · Yajun Deng
Quoted 199 lines · David Hildenbrand
Quoted 86 lines · Mike Rapoport
Quoted 49 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY
context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context,
and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early()    341ms

after:
free_low_memory_core_early()    285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it. v1:
https://lore.kernel.org/all/[email protected]/

---
   mm/mm_init.c    | 18 +++++++++++++-----
   mm/page_alloc.c | 20 ++++++++++++--------
   2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit
 init_reserved_page(unsigned long pfn, int nid)
           if (zone_spans_pfn(zone, pfn))
               break;
       }
-    __init_single_page(pfn_to_page(pfn), pfn, zid, nid,
 INIT_PAGE_COUNT);
+    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
   }

  #else
  static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c void __meminit · line 759

Quoted 5 lines
mm/mm_init.c +1 -1
@@ -756,8 +756,8 @@ void __meminit
 reserve_bootmem_region(phys_addr_t start,
                 init_reserved_page(start_pfn, nid);
   -            /* Avoid false-positive PageTail() */
-            INIT_LIST_HEAD(&page->lru);
+            /* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 13 lines
mm/mm_init.c +1 -0
+            init_page_count(page);
                 /*
                * no need for atomic set_bit because the struct
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long

size, int nid, unsigned long zone
          }
            page = pfn_to_page(pfn);
-        __init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT); -        if (context == MEMINIT_HOTPLUG)
+
+        /* If the context is MEMINIT_EARLY, we will init page count and
+         * mark page reserved in reserve_bootmem_region, the free

Quoted 8 lines

region
+         * wouldn't have page count and we will check the pages count +         * in __free_pages_core.
+         */
+        __init_single_page(page, pfn, zone, nid, 0);
+        if (context == MEMINIT_HOTPLUG) {
+            init_page_count(page);
              __SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED
an call __init_single_page() after the check for MEMINIT_HOTPLUG.

Quoted 3 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all.
@David, can you comment please?

There are a lot of details to that, and I'll share some I can briefly think of.

1) __SetPageReserved()

I tried removing that a while ago, but there was a blocker (IIRC something about
ZONE_DEVICE). I still have the patches at [1] and I could probably take a look
if that blocker still exists (I recall that something changed at some point, but
I never had the time to follow up).

But once we stop setting the pages reserved, we might run into issues with ...

2) init_page_count()

virtio-mem, XEN balloon and HV-balloon add memory blocks that can contain holes.
set_online_page_callback() is used to intercept memory onlining and to expose
only the pages that are not holes to the buddy: calling generic_online_page() on !hole.

Holes are PageReserved but with an initialized page count. Memory offlining will fail on
PageReserved pages -- has_unmovable_pages().

At least virtio-mem clears the PageReserved flag of holes when onlining memory,
and currently relies in the page count to be reasonable (so memory offlining can work).

static void virtio_mem_set_fake_offline(unsigned long pfn,
                    unsigned long nr_pages, bool onlined)
{
    page_offline_begin();
    for (; nr_pages--; pfn++) {
        struct page *page = pfn_to_page(pfn);

        __SetPageOffline(page);
        if (!onlined) {
            SetPageDirty(page);
            /* FIXME: remove after cleanups */
            ClearPageReserved(page);
        }
    }
    page_offline_end();
}

For virtio-mem, we could initialize the page count there instead. The other PV drivers
might require a bit more thought.

[1]
https://github.com/davidhildenbrand/linux/tree/online_reserved_cleanup

Quoted 20 lines
Quoted 20 lines

+        }
            /*
           * Usually, we want to mark the pageblock MIGRATE_MOVABLE,

mm/page_alloc.c +0 -1
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page,
 unsigned int order)
       unsigned int loop;
         /*
-     * When initializing the memmap, __init_single_page() sets the

refcount
-     * of all pages to 1 ("allocated"/"not free"). We have to set the -     * refcount of all involved pages to 0.
+     * When initializing the memmap, memmap_init_range sets the refcount
+     * of all pages to 1 ("reserved" and "free") in hotplug context. We

Quoted 21 lines
Quoted 6 lines

+     * have to set the refcount of all involved pages to 0. Otherwise,
+     * we don't do it, as reserve_bootmem_region only set the refcount on
+     * reserve region ("reserved") in early context.
       */

Again, why hotplug and early init should be different?

Quoted 12 lines

-    prefetchw(p);
-    for (loop = 0; loop < (nr_pages - 1); loop++, p++) { -        prefetchw(p + 1);
+    if (page_count(page)) {
+        prefetchw(p);
+        for (loop = 0; loop < (nr_pages - 1); loop++, p++) { +            prefetchw(p + 1);
+            __ClearPageReserved(p);
+            set_page_count(p, 0);
+        }
          __ClearPageReserved(p); set_page_count(p, 0);

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory,
you wouldn't clear the reserved flag.

Quoted 6 lines
Quoted 1 line

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

Quoted 7 lines

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

	if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Signature · 3 lines
Cheers,

David / dhildenb
author
#24 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 217 lines · David Hildenbrand
Quoted 216 lines · Yajun Deng
Quoted 204 lines · Yajun Deng
Quoted 202 lines · David Hildenbrand
Quoted 88 lines · Mike Rapoport
Quoted 52 lines · Yajun Deng

memmap_init_range() would init page count of all pages, but the free pages count would be reset in __free_pages_core(). There are opposite operations. It's unnecessary and time-consuming when it's MEMINIT_EARLY
context.

Init page count in reserve_bootmem_region when in MEMINIT_EARLY context,
and check the page count before reset it.

At the same time, the INIT_LIST_HEAD in reserve_bootmem_region isn't need, as it already done in __init_single_page.

The following data was tested on an x86 machine with 190GB of RAM.

before:
free_low_memory_core_early()    341ms

after:
free_low_memory_core_early()    285ms

---
v4: same with v2.
v3: same with v2.
v2: check page count instead of check context before reset it. v1:
https://lore.kernel.org/all/[email protected]/

---
   mm/mm_init.c    | 18 +++++++++++++-----
   mm/page_alloc.c | 20 ++++++++++++--------

   2 files changed, 25 insertions(+), 13 deletions(-)
mm/mm_init.c +1 -1
index 9716c8a7ade9..3ab8861e1ef3 100644
@@ -718,7 +718,7 @@ static void __meminit
 init_reserved_page(unsigned long pfn, int nid)
            if (zone_spans_pfn(zone, pfn))
                break;
        }
-    __init_single_page(pfn_to_page(pfn), pfn, zid, nid,
 INIT_PAGE_COUNT);
+    __init_single_page(pfn_to_page(pfn), pfn, zid, nid, 0);
    }

   #else
   static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}

mm/mm_init.c +0 -0
@@ -756,8 +756,8 @@ void __meminit
 reserve_bootmem_region(phys_addr_t start,
                  init_reserved_page(start_pfn, nid);
Quoted 3 lines

   -            /* Avoid false-positive PageTail() */ -            INIT_LIST_HEAD(&page->lru);
+            /* Init page count for reserved region */

Please add a comment that describes _why_ we initialize the page count here.

Quoted 14 lines

+ init_page_count(page);
                 /*
                * no need for atomic set_bit because the struct

mm/mm_init.c +0 -0
@@ -888,9 +888,17 @@ void __meminit memmap_init_range(unsigned long

size, int nid, unsigned long zone
           }
             page = pfn_to_page(pfn);
-        __init_single_page(page, pfn, zone, nid, INIT_PAGE_COUNT); -        if (context == MEMINIT_HOTPLUG)
+
+        /* If the context is MEMINIT_EARLY, we will init page count and
+         * mark page reserved in reserve_bootmem_region, the free region

Quoted 8 lines

+         * wouldn't have page count and we will check the pages count
+         * in __free_pages_core.
+         */
+        __init_single_page(page, pfn, zone, nid, 0); +        if (context == MEMINIT_HOTPLUG) {
+            init_page_count(page); __SetPageReserved(page);

Rather than calling init_page_count() and __SetPageReserved() for MEMINIT_HOTPLUG you can set flags to INIT_PAGE_COUNT | INIT_PAGE_RESERVED
an call __init_single_page() after the check for MEMINIT_HOTPLUG.

Quoted 3 lines

But more generally, I wonder if we have to differentiate HOTPLUG here at all.
@David, can you comment please?

There are a lot of details to that, and I'll share some I can briefly think of.

1) __SetPageReserved()

I tried removing that a while ago, but there was a blocker (IIRC something about
ZONE_DEVICE). I still have the patches at [1] and I could probably take a look
if that blocker still exists (I recall that something changed at some point, but
I never had the time to follow up).

But once we stop setting the pages reserved, we might run into issues with ...

2) init_page_count()

virtio-mem, XEN balloon and HV-balloon add memory blocks that can contain holes.
set_online_page_callback() is used to intercept memory onlining and to expose
only the pages that are not holes to the buddy: calling generic_online_page() on !hole.

Holes are PageReserved but with an initialized page count. Memory offlining will fail on
PageReserved pages -- has_unmovable_pages().

At least virtio-mem clears the PageReserved flag of holes when onlining memory,
and currently relies in the page count to be reasonable (so memory offlining can work).

static void virtio_mem_set_fake_offline(unsigned long pfn, unsigned long nr_pages, bool onlined) {
     page_offline_begin();
     for (; nr_pages--; pfn++) {
         struct page *page = pfn_to_page(pfn);

         __SetPageOffline(page);
         if (!onlined) {
             SetPageDirty(page);
             /* FIXME: remove after cleanups */
             ClearPageReserved(page);
         }
     }
     page_offline_end();
}

For virtio-mem, we could initialize the page count there instead. The other PV drivers
might require a bit more thought.

[1]
https://github.com/davidhildenbrand/linux/tree/online_reserved_cleanup

Quoted 21 lines
Quoted 21 lines

+        }
             /*
            * Usually, we want to mark the pageblock MIGRATE_MOVABLE,

mm/page_alloc.c +0 -1
index 06be8821d833..b868caabe8dc 100644
@@ -1285,18 +1285,22 @@ void __free_pages_core(struct page *page,
 unsigned int order)
        unsigned int loop;
          /*
-     * When initializing the memmap, __init_single_page() sets the

refcount
-     * of all pages to 1 ("allocated"/"not free"). We have to set the
-     * refcount of all involved pages to 0.
+     * When initializing the memmap, memmap_init_range sets the refcount
+     * of all pages to 1 ("reserved" and "free") in hotplug context. We

Quoted 21 lines
Quoted 6 lines

+     * have to set the refcount of all involved pages to 0. Otherwise,
+     * we don't do it, as reserve_bootmem_region only set the refcount on
+     * reserve region ("reserved") in early context. */

Again, why hotplug and early init should be different?

Quoted 12 lines

-    prefetchw(p);
-    for (loop = 0; loop < (nr_pages - 1); loop++, p++) { -        prefetchw(p + 1);
+    if (page_count(page)) {
+        prefetchw(p);
+        for (loop = 0; loop < (nr_pages - 1); loop++, p++) { +            prefetchw(p + 1);
+            __ClearPageReserved(p);
+            set_page_count(p, 0);
+        }
           __ClearPageReserved(p); set_page_count(p, 0);

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory,
you wouldn't clear the reserved flag.

Quoted 6 lines
Quoted 1 line

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 7 lines
Quoted 4 lines

It'll be safer, but more complex. Please comment...

    if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

This link is v1, https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

@Mike,  By the way,  this code will cost more time:

                if (context == MEMINIT_HOTPLUG)
                        flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED; __init_single_page(page, pfn, zone, nid, flags);

[    0.014999] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.311560] ACPI: PM-Timer IO Port: 0x508

This code will cost less time:

                __init_single_page(page, pfn, zone, nid, 0); if (context == MEMINIT_HOTPLUG) { init_page_count(page); __SetPageReserved(page);

[    0.014299] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.250223] ACPI: PM-Timer IO Port: 0x508

maintainer
#25 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 29 lines · Yajun Deng
Quoted 17 lines · David Hildenbrand
Quoted 16 lines · Yajun Deng
Quoted 10 lines · Yajun Deng
Quoted 6 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 9 lines
Quoted 5 lines

It'll be safer, but more complex. Please comment...

    if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Quoted 5 lines

This link is v1,
https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Quoted 21 lines

@Mike,  By the way,  this code will cost more time:

                if (context == MEMINIT_HOTPLUG)
                        flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED; __init_single_page(page, pfn, zone, nid, flags);

[    0.014999] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.311560] ACPI: PM-Timer IO Port: 0x508

This code will cost less time:

                __init_single_page(page, pfn, zone, nid, 0); if (context == MEMINIT_HOTPLUG) { init_page_count(page); __SetPageReserved(page);

[    0.014299] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.250223] ACPI: PM-Timer IO Port: 0x508

Signature · 2 lines
Sincerely yours,
Mike.
author
#26 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 23 lines · Mike Rapoport
Quoted 22 lines · Yajun Deng
Quoted 21 lines · David Hildenbrand
Quoted 9 lines · Yajun Deng
Quoted 8 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

Quoted 8 lines

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

    if (context != MEMINIT_EARLY || (page_count(page) ||
PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Quoted 8 lines
Quoted 5 lines

This link is v1,
https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508

[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code, 4204K rwdata, 5608K rodata, 3188K init, 17024K bss, 5499616K reserved, 20971520K cma-reserved)

after:

[    0.260229] software IO TLB: area num 32.

[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code, 4204K rwdata, 5608K rodata, 3188K init, 17024K bss, 5499616K reserved, 20971520K cma-reserved)

Memory initialization is time consuming in the boot log.

Quoted 3 lines

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024 times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page)) {'

This is a foundation. We may change this when we are able to safely remove page init in the hotplug context one day.

So the case for the early context is only temporary.

Quoted 21 lines
Quoted 21 lines

@Mike,  By the way,  this code will cost more time:

                if (context == MEMINIT_HOTPLUG)
                        flags = INIT_PAGE_COUNT | INIT_PAGE_RESERVED; __init_single_page(page, pfn, zone, nid, flags);

[    0.014999] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.311560] ACPI: PM-Timer IO Port: 0x508

This code will cost less time:

                __init_single_page(page, pfn, zone, nid, 0); if (context == MEMINIT_HOTPLUG) { init_page_count(page); __SetPageReserved(page);

[    0.014299] On node 0, zone DMA32: 31679 pages in unavailable ranges [    0.250223] ACPI: PM-Timer IO Port: 0x508

maintainer
#27 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 44 lines · Yajun Deng
Quoted 23 lines · Mike Rapoport
Quoted 22 lines · Yajun Deng
Quoted 21 lines · David Hildenbrand
Quoted 9 lines · Yajun Deng
Quoted 8 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

Quoted 8 lines

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

    if (context != MEMINIT_EARLY || (page_count(page) ||
PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Quoted 8 lines
Quoted 5 lines

This link is v1,
https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

Quoted 6 lines

after:

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code,

Memory initialization is time consuming in the boot log.

You just confirmed that 56 msec is insignificant and then you send again the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

Quoted 9 lines
Quoted 2 lines

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024 times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page)) {'

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have to reason what will happen with that condition after the change.

Signature · 2 lines
Sincerely yours,
Mike.
author
#28 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 45 lines · Mike Rapoport
Quoted 44 lines · Yajun Deng
Quoted 23 lines · Mike Rapoport
Quoted 22 lines · Yajun Deng
Quoted 21 lines · David Hildenbrand
Quoted 9 lines · Yajun Deng
Quoted 8 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

Quoted 8 lines

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

    if (context != MEMINIT_EARLY || (page_count(page) ||
PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Quoted 8 lines
Quoted 5 lines

This link is v1,
https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

after:

Quoted 8 lines
Quoted 4 lines

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code, Memory:
Memory initialization is time consuming in the boot log.

You just confirmed that 56 msec is insignificant and then you send again the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 7 lines
Quoted 7 lines
Quoted 3 lines

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024 times. The following 'if' would be safer:

Quoted 8 lines
Quoted 2 lines

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page)) {'

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have to reason what will happen with that condition after the change.

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

if (page_count(page) || PageReserved(page))

#29 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 88 lines · Yajun Deng
Quoted 45 lines · Mike Rapoport
Quoted 44 lines · Yajun Deng
Quoted 23 lines · Mike Rapoport
Quoted 22 lines · Yajun Deng
Quoted 21 lines · David Hildenbrand
Quoted 9 lines · Yajun Deng
Quoted 8 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

Quoted 8 lines

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

It'll be safer, but more complex. Please comment...

       if (context != MEMINIT_EARLY || (page_count(page) ||

PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

Quoted 8 lines
Quoted 5 lines

This link is v1,
https://lore.kernel.org/all/[email protected]/

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

after:

Quoted 8 lines
Quoted 4 lines

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code, Memory:
Memory initialization is time consuming in the boot log.

You just confirmed that 56 msec is insignificant and then you send again the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 7 lines
Quoted 7 lines
Quoted 3 lines

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024 times. The following 'if' would be safer:

Quoted 8 lines
Quoted 2 lines

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page)) {'

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have to reason what will happen with that condition after the change.

Quoted 4 lines

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

if (page_count(page) || PageReserved(page))

I'll have to side with Mike here; this change might not be worth it.

Signature · 3 lines
Cheers,

David / dhildenb
author
#30 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 89 lines · David Hildenbrand
Quoted 88 lines · Yajun Deng
Quoted 46 lines · Mike Rapoport
Quoted 45 lines · Yajun Deng
Quoted 27 lines · Mike Rapoport
Quoted 26 lines · Yajun Deng
Quoted 13 lines · David Hildenbrand
Quoted 12 lines · Yajun Deng
Quoted 8 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I need to do something else?

How about the following if statement? But it needs to add more patch like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 8 lines
Quoted 5 lines

It'll be safer, but more complex. Please comment...

       if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not check for count or the reserved flag.

This link is v1,
https://lore.kernel.org/all/[email protected]/

Quoted 5 lines
Quoted 2 lines

If we could make initialization only depend on the context, I'll modify it based on v1.

Although ~20% improvement looks impressive, this is only optimization of a fraction of the boot time, and realistically, how much 56 msec saves from the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

after:

Quoted 8 lines
Quoted 4 lines

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code, Memory:
Memory initialization is time consuming in the boot log.

You just confirmed that 56 msec is insignificant and then you send again the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 14 lines
Quoted 8 lines
Quoted 2 lines

I still think the improvement does not justify the churn, added complexity and special casing of different code paths of initialization of struct pages.

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024 times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page)) {'

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have to reason what will happen with that condition after the change.

Quoted 7 lines
Quoted 4 lines

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

if (page_count(page) || PageReserved(page))

I'll have to side with Mike here; this change might not be worth it.

Okay, I got it. Thanks!

#31 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 111 lines · Yajun Deng
Quoted 109 lines · David Hildenbrand
Quoted 108 lines · Yajun Deng
Quoted 60 lines · Mike Rapoport
Quoted 52 lines · Yajun Deng
Quoted 30 lines · Mike Rapoport
Quoted 29 lines · Yajun Deng
Quoted 15 lines · David Hildenbrand
Quoted 14 lines · Yajun Deng
Quoted 9 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I
need to do something else?

How about the following if statement? But it needs to add more patch
like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 9 lines
Quoted 5 lines

It'll be safer, but more complex. Please comment...

       if (context != MEMINIT_EARLY || (page_count(page) ||
PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not
check for count or the reserved flag.

This link is v1,
https://lore.kernel.org/all/[email protected]/

Quoted 8 lines
Quoted 3 lines

If we could make initialization only depend on the context, I'll modify it
based on v1.

Although ~20% improvement looks impressive, this is only optimization of a
fraction of the boot time, and realistically, how much 56 msec saves from
the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec
saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

Quoted 7 lines

after:

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code,
Memory:
Memory initialization is time consuming in the boot log.

Quoted 5 lines

You just confirmed that 56 msec is insignificant and then you send again
the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines

I still think the improvement does not justify the churn, added complexity
and special casing of different code paths of initialization of struct pages.

Quoted 7 lines

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024
times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page))
{'

Quoted 8 lines

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes
the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have
to reason what will happen with that condition after the change.

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

if (page_count(page) || PageReserved(page))

I'll have to side with Mike here; this change might not be worth it.

Okay, I got it. Thanks!

IMHO instead of adding more checks to that code we should try to unify that handling such that we can just remove it. As expressed, at least from the memory hotplug perspective there are still reasons why we need that; I can provide some guidance on how to eventually achieve that, but it might end up in a bit of work ...

Anyhow, thanks for bringing up that topic; it reminded me that I still have pending cleanups to not rely on PageReserved on the memory hotplug path.

Signature · 3 lines
Cheers,

David / dhildenb
author
#32 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 113 lines · David Hildenbrand
Quoted 112 lines · Yajun Deng
Quoted 110 lines · David Hildenbrand
Quoted 109 lines · Yajun Deng
Quoted 61 lines · Mike Rapoport
Quoted 53 lines · Yajun Deng
Quoted 31 lines · Mike Rapoport
Quoted 30 lines · Yajun Deng
Quoted 15 lines · David Hildenbrand
Quoted 14 lines · Yajun Deng
Quoted 9 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0 already for hotplugged memory, you wouldn't clear the reserved flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I
need to do something else?

How about the following if statement? But it needs to add more patch
like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 9 lines
Quoted 5 lines

It'll be safer, but more complex. Please comment...

        if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not
check for count or the reserved flag.

This link is v1,
https://lore.kernel.org/all/[email protected]/

Quoted 8 lines
Quoted 3 lines

If we could make initialization only depend on the context, I'll modify it
based on v1.

Although ~20% improvement looks impressive, this is only optimization of a
fraction of the boot time, and realistically, how much 56 msec saves from
the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec
saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

Quoted 7 lines

after:

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code,
Memory:
Memory initialization is time consuming in the boot log.

Quoted 5 lines

You just confirmed that 56 msec is insignificant and then you send again
the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines

I still think the improvement does not justify the churn, added complexity
and special casing of different code paths of initialization of struct pages.

Quoted 7 lines

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024
times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page))
{'

Quoted 8 lines

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes
the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have
to reason what will happen with that condition after the change.

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

if (page_count(page) || PageReserved(page))

I'll have to side with Mike here; this change might not be worth it.

Okay, I got it. Thanks!

IMHO instead of adding more checks to that code we should try to unify that handling such that we can just remove it. As expressed, at least from the memory hotplug perspective there are still reasons why we need that; I can provide some guidance on how to eventually achieve that, but it might end up in a bit of work ...

Yes, we can't remove it right now. If we want to do that, we have to clean up rely on page count and PageReserved first.

Quoted 3 lines

Anyhow, thanks for bringing up that topic; it reminded me that I still have pending cleanups to not rely on PageReserved on the memory hotplug path.

author
#33 lore raw

Re: [PATCH v4 2/2] mm: Init page count in reserve_bootmem_region when MEMINIT_EARLY

Quoted 120 lines · Yajun Deng
Quoted 118 lines · David Hildenbrand
Quoted 117 lines · Yajun Deng
Quoted 114 lines · David Hildenbrand
Quoted 106 lines · Yajun Deng
Quoted 61 lines · Mike Rapoport
Quoted 53 lines · Yajun Deng
Quoted 31 lines · Mike Rapoport
Quoted 30 lines · Yajun Deng
Quoted 15 lines · David Hildenbrand
Quoted 14 lines · Yajun Deng
Quoted 9 lines · Yajun Deng
Quoted 5 lines

That looks wrong. if the page count would by pure luck be 0
already for hotplugged memory, you wouldn't clear the reserved
flag.

These changes make me a bit nervous.

Is 'if (page_count(page) || PageReserved(page))' be safer? Or do I
need to do something else?

How about the following if statement? But it needs to add more patch
like v1 ([PATCH 2/4] mm: Introduce MEMINIT_LATE context).

Quoted 9 lines
Quoted 5 lines

It'll be safer, but more complex. Please comment...

        if (context != MEMINIT_EARLY || (page_count(page) || PageReserved(page)) {

Ideally we could make initialization only depend on the context, and not
check for count or the reserved flag.

This link is v1,
https://lore.kernel.org/all/[email protected]/

Quoted 8 lines
Quoted 3 lines

If we could make initialization only depend on the context, I'll modify it
based on v1.

Although ~20% improvement looks impressive, this is only optimization of a
fraction of the boot time, and realistically, how much 56 msec saves from
the total boot time when you boot a machine with 190G of RAM?

There are a lot of factors that can affect the total boot time. 56 msec
saves may be insignificant.

But if we look at the boot log, we'll see there's a significant time jump.

before:

[    0.250334] ACPI: PM-Timer IO Port: 0x508
[    0.618994] Memory: 173413056K/199884452K available (18440K kernel code,

Quoted 7 lines

after:

[    0.260229] software IO TLB: area num 32.
[    0.563497] Memory: 173413056K/199884452K available (18440K kernel code,
Memory:
Memory initialization is time consuming in the boot log.

Quoted 5 lines

You just confirmed that 56 msec is insignificant and then you send again
the improvement of ~60 msec in memory initialization.

What does this improvement gain in percentage of total boot time?

before:

[   10.692708] Run /init as init process

after:

[   10.666290] Run /init as init process

About 0.25%. The total boot time is variable, depending on how many drivers need to be initialized.

Quoted 12 lines
Quoted 5 lines
Quoted 4 lines

I still think the improvement does not justify the churn, added complexity
and special casing of different code paths of initialization of struct pages.

Quoted 7 lines

Because there is a loop, if the order is MAX_ORDER, the loop will run 1024
times. The following 'if' would be safer:

'if (context != MEMINIT_EARLY || (page_count(page) || >> PageReserved(page))
{'

Quoted 8 lines

No, it will not.

As the matter of fact any condition here won't be 'safer' because it makes
the code more complex and less maintainable.
Any future change in __free_pages_core() or one of it's callers will have
to reason what will happen with that condition after the change.

Quoted 4 lines

To avoid introducing MEMINIT_LATE context and make code simpler. This might be a better option.

if (page_count(page) || PageReserved(page))

I'll have to side with Mike here; this change might not be worth it.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Okay, I got it. Thanks!

IMHO instead of adding more checks to that code we should try to unify that handling such that we can just remove it. As expressed, at least from the memory hotplug perspective there are still reasons why we need that; I can provide some guidance on how to eventually achieve that, but it might end up in a bit of work ...

Yes, we can't remove it right now. If we want to do that, we have to clean up rely on page count and PageReserved first.

How about making __free_pages_core separate, like:

void __init __free_pages_core_early(struct page *page, unsigned int order) {
        unsigned int nr_pages = 1 << order;

        atomic_long_add(nr_pages, &page_zone(page)->managed_pages);

        if (page_contains_unaccepted(page, order)) {
                if (order == MAX_ORDER && __free_unaccepted(page)) return;

                accept_page(page, order);
        }

        /*
         * Bypass PCP and place fresh pages right to the tail, primarily
         * relevant for memory onlining.
         */
        __free_pages_ok(page, order, FPI_TO_TAIL);
}

void __free_pages_core(struct page *page, unsigned int order) {
        unsigned int nr_pages = 1 << order;
        struct page *p = page;
        unsigned int loop;

        /*
         * When initializing the memmap, __init_single_page() sets the refcount
         * of all pages to 1 ("allocated"/"not free"). We have to set the
         * refcount of all involved pages to 0.
         */
        prefetchw(p);
        for (loop = 0; loop < (nr_pages - 1); loop++, p++) { prefetchw(p + 1);
                __ClearPageReserved(p);
                set_page_count(p, 0);
        }
        __ClearPageReserved(p);
        set_page_count(p, 0);

        __free_pages_core_early(page, order);
}

We only change the caller we need to __free_pages_core_early, it doesn't affect other callers.

Quoted 4 lines
Quoted 4 lines

Anyhow, thanks for bringing up that topic; it reminded me that I still have pending cleanups to not rely on PageReserved on the memory hotplug path.