memory_hotplug: Free pages as higher order
Subsystems (6)
PATCH v4 1/1 memory_hotplug: Free pages as higher order
When free pages are done with higher order, time spend on coalescing pages by buddy allocator can be reduced. With section size of 256MB, hot add latency of a single section shows improvement from 50-60 ms to less than 1 ms, hence improving the hot add latency by 60%. Modify external providers of online callback to align with the change. Also remove prefetch from __free_pages_core().
---
Changes since v3:
- renamed _free_pages_boot_core -> __free_pages_core.
- removed prefetch from __free_pages_core.
- removed xen_online_page().
Changes since v2:
- reuse code from __free_pages_boot_core().
Changes since v1:
- Removed prefetch().
Changes since RFC:
- Rebase.
- As suggested by Michal Hocko remove pages_per_block.
- Modifed external providers of online_page_callback.
v3: https://lore.kernel.org/patchwork/patch/992348/
v2: https://lore.kernel.org/patchwork/patch/991363/
v1: https://lore.kernel.org/patchwork/patch/989445/
RFC: https://lore.kernel.org/patchwork/patch/984754/
--- drivers/hv/hv_balloon.c | 6 ++++-- drivers/xen/balloon.c | 23 ++++++++++++++-------- include/linux/memory_hotplug.h | 2 +- mm/internal.h | 1 + mm/memory_hotplug.c | 44 ++++++++++++++++++++++++++++++------------ mm/page_alloc.c | 14 +++++--------- 6 files changed, 58 insertions(+), 32 deletions(-)
drivers/hv/hv_balloon.c +4 -2
@@ -771,7 +771,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size, } } -static void hv_online_page (struct page *pg) +static int hv_online_page (struct page *pg,unsigned int order) {struct hv_hotadd_state *has;unsigned long flags;
@@ -783,10 +783,12 @@ static void hv_online_page(struct page *pg)if ((pfn < has->start_pfn) || (pfn >= has->end_pfn))continue ; -hv_page_online_one (has, pg); +hv_bring_pgs_online (has, pfn, (1UL << order));break ; }spin_unlock_irqrestore (&dm_device.ha_lock, flags); + +return 0 ; }static int pfn_covered (unsigned long start_pfn,unsigned long pfn_cnt)
drivers/xen/balloon.c +15 -8
@@ -390,8 +390,8 @@ static enum bp_state reserve_additional_memory(void)/* * add_memory_resource() will call online_pages() which in its turn -* will call xen_online_page() callback causing deadlock if we don't -* release balloon_mutex here. Unlocking here is safe because the +* will call xen_bring_pgs_online() callback causing deadlock if we +* don't release balloon_mutex here. Unlocking here is safe because the * callers drop the mutex before trying again. */ mutex_unlock (&balloon_mutex);
@@ -411,15 +411,22 @@ static enum bp_state reserve_additional_memory(void)return BP_ECANCELED ; } -static void xen_online_page (struct page *page) +static int xen_bring_pgs_online (struct page *pg,unsigned int order) { -__online_page_set_limits (page); +unsigned long i, size = (1 << order); +unsigned long start_pfn =page_to_pfn (pg); +struct page *p; +pr_debug ("Online %lu pages starting at pfn 0x%lx\n" , size, start_pfn);mutex_lock (&balloon_mutex); - -__balloon_append (page); - +for (i =0 ; i < size; i++) { + p =pfn_to_page (start_pfn + i); +__online_page_set_limits (p); +__balloon_append (p); + }mutex_unlock (&balloon_mutex); + +return 0 ; }static int xen_memory_notifier (struct notifier_block *nb,unsigned long val,void *v)
@@ -744,7 +751,7 @@ static int __init balloon_init(void) balloon_stats.max_retry_count =RETRY_UNLIMITED ;#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG -set_online_page_callback (&xen_online_page); +set_online_page_callback (&xen_bring_pgs_online);register_memory_notifier (&xen_memory_nb);register_sysctl_table (xen_root);
include/linux/memory_hotplug.h +1 -1
@@ -87,7 +87,7 @@ extern int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, unsigned long *valid_start, unsigned long *valid_end);extern void __offline_isolated_pages (unsigned long ,unsigned long ); -typedef void (*online_page_callback_t )(struct page *page); +typedef int (*online_page_callback_t )(struct page *page,unsigned int order);extern int set_online_page_callback (online_page_callback_t callback);extern int restore_online_page_callback (online_page_callback_t callback);
mm/internal.h +1 -0
@@ -163,6 +163,7 @@ static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,extern int __isolate_free_page (struct page *page,unsigned int order);extern void __free_pages_bootmem (struct page *page,unsigned long pfn,unsigned int order); +extern void __free_pages_core (struct page *page,unsigned int order);extern void prep_compound_page (struct page *page,unsigned int order);extern void post_alloc_hook (struct page *page,unsigned int order,gfp_t gfp_flags);
mm/memory_hotplug.c +32 -12
65 lines not on this page · raw message
mm/page_alloc.c +5 -9
@@ -1252,20 +1252,16 @@ static void __free_pages_ok(struct page *page, unsigned int order)local_irq_restore (flags); } -static void __init __free_pages_boot_core (struct page *page,unsigned int order) +void __free_pages_core (struct page *page,unsigned int order) {unsigned int nr_pages =1 << order;struct page *p = page;unsigned int loop; -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { -prefetchw (p +1 ); +for (loop =0 ; loop < nr_pages; loop++, p++) {__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );page_zone (page)->managed_pages += nr_pages;set_page_refcounted (page);
@@ -1331,7 +1327,7 @@ void __init __free_pages_bootmem(struct page *page, unsigned long pfn, {if (early_page_uninitialised (pfn))return ; -return __free_pages_boot_core (page, order); +return __free_pages_core (page, order); }/*
@@ -1421,14 +1417,14 @@ static void __init deferred_free_range(unsigned long pfn,if (nr_pages == pageblock_nr_pages && (pfn & (pageblock_nr_pages -1 )) ==0 ) {set_pageblock_migratetype (page,MIGRATE_MOVABLE ); -__free_pages_boot_core (page, pageblock_order); +__free_pages_core (page, pageblock_order);return ; }for (i =0 ; i < nr_pages; i++, page++, pfn++) {if ((pfn & (pageblock_nr_pages -1 )) ==0 )set_pageblock_migratetype (page,MIGRATE_MOVABLE ); -__free_pages_boot_core (page,0 ); +__free_pages_core (page,0 ); } }
Signature · 1 line
1.9.1
On Wed 03-10-18 19:09:39, Arun KS wrote:
[...]
mm/memory_hotplug.c void __online_page_free(struct page *page) · line 678
Quoted 10 lines
mm/memory_hotplug.c +10
-0+static int online_pages_blocks (unsigned long start,unsigned long nr_pages) +{ +unsigned long end = start + nr_pages; +int order, ret, onlined_pages =0 ; + +while (start < end) { + order =min (MAX_ORDER -1UL ,__ffs (start)); + +while (start + (1UL << order) > end) + order--;
this really made me scratch my head. Wouldn't it be much simpler to do the following?
order = min(MAX_ORDER - 1, get_order(end - start))?
Quoted 11 lines
mm/memory_hotplug.c +10
-0+ + ret = (*online_page_callback)(pfn_to_page (start), order); +if (!ret) + onlined_pages += (1UL << order); +else if (ret >0 ) + onlined_pages += ret; + + start += (1UL << order); + } +return onlined_pages; }
[...]
Quoted 9 lines
mm/page_alloc.c +1
-3-static void __init __free_pages_boot_core (struct page *page,unsigned int order) +void __free_pages_core (struct page *page,unsigned int order) {unsigned int nr_pages =1 << order;struct page *p = page;unsigned int loop; -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) {
mm/page_alloc.c static void __free_pages_ok(struct page *page, unsigned int order) · line 1267
Quoted 10 lines
mm/page_alloc.c +1
-3-prefetchw (p +1 ); +for (loop =0 ; loop < nr_pages; loop++, p++) {__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );page_zone (page)->managed_pages += nr_pages;set_page_refcounted (page);
I think this is wort a separate patch as it is unrelated to the patch.
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 17 lines · Michal Hocko
On Wed 03-10-18 19:09:39, Arun KS wrote:
[...]mm/memory_hotplug.c void __online_page_free(struct page *page) · line 678
Quoted 11 lines
+static int
online_pages_blocks(unsigned long start, unsigned long nr_pages)mm/memory_hotplug.c +9
-0+{ +unsigned long end = start + nr_pages; +int order, ret, onlined_pages =0 ; + +while (start < end) { + order =min (MAX_ORDER -1UL ,__ffs (start)); + +while (start + (1UL << order) > end) + order--;this really made me scratch my head. Wouldn't it be much simpler to do the following?
order =min(MAX_ORDER - 1, get_order(end - start))?
Yes. Much better. Will change to,
order = min(MAX_ORDER - 1,
get_order(PFN_PHYS(end) - PFN_PHYS(start)));
Quoted 31 lines
Quoted 12 lines
mm/memory_hotplug.c +10
-0+ + ret = (*online_page_callback)(pfn_to_page (start), order); +if (!ret) + onlined_pages += (1UL << order); +else if (ret >0 ) + onlined_pages += ret; + + start += (1UL << order); + } +return onlined_pages; }[...]
Quoted 18 lines
-static void
__init__free_pages_boot_core(struct page *page, unsigned int order)mm/page_alloc.c +2
-5+void __free_pages_core (struct page *page,unsigned int order) {unsigned int nr_pages =1 << order;struct page *p = page;unsigned int loop; -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { -prefetchw (p +1 ); +for (loop =0 ; loop < nr_pages; loop++, p++) {__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );
Quoted 4 lines
mm/page_alloc.c static void __free_pages_ok(struct page *page, unsigned int order) · line 1267
Quoted 2 lines
mm/page_alloc.c +0
-0page_zone (page)->managed_pages += nr_pages;set_page_refcounted (page);I think this is wort a separate patch as it is unrelated to the patch.
Sure. Will split the patch.
Regards,
Arun
Quoted 17 lines · Michal Hocko
On Wed 03-10-18 19:09:39, Arun KS wrote:
[...]mm/memory_hotplug.c void __online_page_free(struct page *page) · line 678
Quoted 11 lines
+static int
online_pages_blocks(unsigned long start, unsigned long nr_pages)mm/memory_hotplug.c +9
-0+{ +unsigned long end = start + nr_pages; +int order, ret, onlined_pages =0 ; + +while (start < end) { + order =min (MAX_ORDER -1UL ,__ffs (start)); + +while (start + (1UL << order) > end) + order--;this really made me scratch my head. Wouldn't it be much simpler to do the following?
order =min(MAX_ORDER - 1, get_order(end - start))?
Yes. Much better. Will change to,
order = min(MAX_ORDER - 1,
get_order(PFN_PHYS(end) - PFN_PHYS(start)));
Quoted 31 lines
Quoted 12 lines
mm/memory_hotplug.c +10
-0+ + ret = (*online_page_callback)(pfn_to_page (start), order); +if (!ret) + onlined_pages += (1UL << order); +else if (ret >0 ) + onlined_pages += ret; + + start += (1UL << order); + } +return onlined_pages; }[...]
Quoted 18 lines
-static void
__init__free_pages_boot_core(struct page *page, unsigned int order)mm/page_alloc.c +2
-5+void __free_pages_core (struct page *page,unsigned int order) {unsigned int nr_pages =1 << order;struct page *p = page;unsigned int loop; -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) { -prefetchw (p +1 ); +for (loop =0 ; loop < nr_pages; loop++, p++) {__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );
Quoted 4 lines
mm/page_alloc.c static void __free_pages_ok(struct page *page, unsigned int order) · line 1267
Quoted 2 lines
mm/page_alloc.c +0
-0page_zone (page)->managed_pages += nr_pages;set_page_refcounted (page);I think this is wort a separate patch as it is unrelated to the patch.
Sure. Will split the patch.
Regards,
Arun
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
On Wed 03-10-18 19:09:39, Arun KS wrote:
[...]
mm/memory_hotplug.c void __online_page_free(struct page *page) · line 678
Quoted 10 lines
mm/memory_hotplug.c +10
-0+static int online_pages_blocks (unsigned long start,unsigned long nr_pages) +{ +unsigned long end = start + nr_pages; +int order, ret, onlined_pages =0 ; + +while (start < end) { + order =min (MAX_ORDER -1UL ,__ffs (start)); + +while (start + (1UL << order) > end) + order--;
this really made me scratch my head. Wouldn't it be much simpler to do the following?
order = min(MAX_ORDER - 1, get_order(end - start))?
Quoted 11 lines
mm/memory_hotplug.c +10
-0+ + ret = (*online_page_callback)(pfn_to_page (start), order); +if (!ret) + onlined_pages += (1UL << order); +else if (ret >0 ) + onlined_pages += ret; + + start += (1UL << order); + } +return onlined_pages; }
[...]
Quoted 9 lines
mm/page_alloc.c +1
-3-static void __init __free_pages_boot_core (struct page *page,unsigned int order) +void __free_pages_core (struct page *page,unsigned int order) {unsigned int nr_pages =1 << order;struct page *p = page;unsigned int loop; -prefetchw (p); -for (loop =0 ; loop < (nr_pages -1 ); loop++, p++) {
mm/page_alloc.c static void __free_pages_ok(struct page *page, unsigned int order) · line 1267
Quoted 10 lines
mm/page_alloc.c +1
-3-prefetchw (p +1 ); +for (loop =0 ; loop < nr_pages; loop++, p++) {__ClearPageReserved (p);set_page_count (p,0 ); } -__ClearPageReserved (p); -set_page_count (p,0 );page_zone (page)->managed_pages += nr_pages;set_page_refcounted (page);
I think this is wort a separate patch as it is unrelated to the patch.
Signature · 7 lines
Michal Hocko SUSE Labs _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel