mm: page-isolation: skip isolated pageblock in start_isolate_page_range()
Subsystems (3)
PATCH 1/2 mm: page-isolation: skip isolated pageblock in start_isolate_page_range()
From: Zi Yan <[email protected]>
start_isolate_page_range() first isolates the first and the last pageblocks in the range and ensure pages across range boundaries are split during isolation. But it missed the case when the range is <= a pageblock and the first and the last pageblocks are the same one, so the second isolate_single_pageblock() will always fail. To fix it, skip the pageblock isolation in second isolate_single_pageblock().
--- mm/page_isolation.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-)
mm/page_isolation.c +18 -8
@@ -300,7 +300,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages)* the in-use page then splitting the free page. */ static int isolate_single_pageblock (unsigned long boundary_pfn,int flags, -gfp_t gfp_flags,bool isolate_before) +gfp_t gfp_flags,bool isolate_before,bool skip_isolation) {unsigned char saved_mt;unsigned long start_pfn;
@@ -327,11 +327,16 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, zone->zone_start_pfn); saved_mt =get_pageblock_migratetype (pfn_to_page (isolate_pageblock)); - ret =set_migratetype_isolate (pfn_to_page (isolate_pageblock), saved_mt, flags, - isolate_pageblock, isolate_pageblock + pageblock_nr_pages); -if (ret) -return ret; +if (skip_isolation) +VM_BUG_ON (!is_migrate_isolate (saved_mt)); +else { + ret =set_migratetype_isolate (pfn_to_page (isolate_pageblock), saved_mt, flags, + isolate_pageblock, isolate_pageblock + pageblock_nr_pages); + +if (ret) +return ret; + }/* * Bail out early when the to-be-isolated pageblock does not form
@@ -463,7 +468,8 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,return 0 ; failed:/* restore the original migratetype */ -unset_migratetype_isolate (pfn_to_page (isolate_pageblock), saved_mt); +if (!skip_isolation) +unset_migratetype_isolate (pfn_to_page (isolate_pageblock), saved_mt);return -EBUSY ; }
@@ -522,14 +528,18 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,unsigned long isolate_start =ALIGN_DOWN (start_pfn, pageblock_nr_pages);unsigned long isolate_end =ALIGN (end_pfn, pageblock_nr_pages);int ret; +bool skip_isolation = false;/* isolate [isolate_start, isolate_start + pageblock_nr_pages) pageblock */ - ret =isolate_single_pageblock (isolate_start, flags, gfp_flags, false); + ret =isolate_single_pageblock (isolate_start, flags, gfp_flags, false, skip_isolation);if (ret)return ret; +if (isolate_start == isolate_end - pageblock_nr_pages) + skip_isolation = true; +/* isolate [isolate_end - pageblock_nr_pages, isolate_end) pageblock */ - ret =isolate_single_pageblock (isolate_end, flags, gfp_flags, true); + ret =isolate_single_pageblock (isolate_end, flags, gfp_flags, true, skip_isolation);if (ret) {unset_migratetype_isolate (pfn_to_page (isolate_start), migratetype);return ret;
Signature · 1 line
2.35.1
PATCH 2/2 mm: split free page with properly free memory accounting and without race
From: Zi Yan <[email protected]>
In isolate_single_pageblock(), free pages are checked without holding zone lock, but they can go away in split_free_page() when zone lock is held. Check the free page and its order again in split_free_page() when zone lock is held. Recheck the page if the free page is gone under zone lock.
In addition, in split_free_page(), the free page was deleted from the page list without changing free page accounting. Add the missing free page accounting code.
Fix the type of order parameter in split_free_page().
--- mm/internal.h | 4 ++-- mm/page_alloc.c | 24 ++++++++++++++++++++---- mm/page_isolation.c | 10 +++++++--- 3 files changed, 29 insertions(+), 9 deletions(-)
mm/internal.h +2 -2
@@ -374,8 +374,8 @@ extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,phys_addr_t min_addr, int nid, bool exact_nid); -void split_free_page (struct page *free_page, -int order,unsigned long split_pfn_offset); +int split_free_page (struct page *free_page, +unsigned int order,unsigned long split_pfn_offset);#if definedCONFIG_COMPACTION || definedCONFIG_CMA
mm/page_alloc.c +20 -4
@@ -1112,30 +1112,44 @@ static inline void __free_one_page(struct page *page,* @order: the order of the page * @split_pfn_offset: split offset within the page * +* Return -ENOENT if the free page is changed, otherwise 0 +* * It is used when the free page crosses two pageblocks with different migratetypes * at split_pfn_offset within the page. The split free page will be put into * separate migratetype lists afterwards. Otherwise, the function achieves * nothing. */ -void split_free_page (struct page *free_page, -int order,unsigned long split_pfn_offset) +int split_free_page (struct page *free_page, +unsigned int order,unsigned long split_pfn_offset) {struct zone *zone =page_zone (free_page);unsigned long free_page_pfn =page_to_pfn (free_page);unsigned long pfn;unsigned long flags;int free_page_order; +int mt; +int ret =0 ;if (split_pfn_offset ==0 ) -return ; +return ret;spin_lock_irqsave (&zone->lock, flags); + +if (!PageBuddy (free_page) ||buddy_order (free_page) != order) { + ret = -ENOENT ; + goto out; + } + + mt =get_pageblock_migratetype (free_page); +if (likely (!is_migrate_isolate (mt))) +__mod_zone_freepage_state (zone, -(1UL << order), mt); +del_page_from_free_list (free_page, zone, order);for (pfn = free_page_pfn; pfn < free_page_pfn + (1UL << order);) {int mt =get_pfnblock_migratetype (pfn_to_page (pfn), pfn); - free_page_order =min_t (int, + free_page_order = min_t(unsigned int, pfn ?__ffs (pfn) : order,__fls (split_pfn_offset)); __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order,
@@ -1146,7 +1160,9 @@ void split_free_page(struct page *free_page,if (split_pfn_offset ==0 ) split_pfn_offset = (1UL << order) - (pfn - free_page_pfn); } +out:spin_unlock_irqrestore (&zone->lock, flags); +return ret; }/* * A bad page could be due to a number of fields. Instead of multiple branches,
mm/page_isolation.c +7 -3
@@ -371,9 +371,13 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,if (PageBuddy (page)) {int order =buddy_order (page); -if (pfn + (1UL << order) > boundary_pfn) -split_free_page (page, order, boundary_pfn - pfn); - pfn += (1UL << order); +if (pfn + (1UL << order) > boundary_pfn) { +/* free page changed before split, check it again */ +if (split_free_page (page, order, boundary_pfn - pfn)) +continue ; + } + + pfn +=1UL << order;continue ; }/*
Signature · 1 line
2.35.1