mm: page_alloc: simplify pageblock migratetype check in __free_one_page().
Subsystems (3)
PATCH v3 1/2 mm: page_alloc: simplify pageblock migratetype check in __free_one_page().
From: Zi Yan <[email protected]>
Move pageblock migratetype check code in the while loop to simplify the logic. It also saves redundant buddy page checking code.
--- mm/page_alloc.c | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-)
mm/page_alloc.c +17 -29
@@ -1054,7 +1054,6 @@ static inline void __free_one_page(struct page *page, int migratetype, fpi_t fpi_flags) {struct capture_control *capc =task_capc (zone); -unsigned int max_order = pageblock_order;unsigned long buddy_pfn;unsigned long combined_pfn;struct page *buddy;
@@ -1070,8 +1069,7 @@ static inline void __free_one_page(struct page *page,VM_BUG_ON_PAGE (pfn & ((1 << order) -1 ), page);VM_BUG_ON_PAGE (bad_range (zone, page), page); -continue_merging: -while (order < max_order) { +while (order <MAX_ORDER -1 ) {if (compaction_capture (capc, page, order, migratetype)) {__mod_zone_freepage_state (zone, -(1 << order), migratetype);
@@ -1082,6 +1080,22 @@ static inline void __free_one_page(struct page *page,if (!page_is_buddy (page, buddy, order)) goto done_merging; + +if (unlikely (order >= pageblock_order)) { +/* +* We want to prevent merge between freepages on pageblock +* without fallbacks and normal pageblock. Without this, +* pageblock isolation could cause incorrect freepage or CMA +* accounting or HIGHATOMIC accounting. +*/ +int buddy_mt =get_pageblock_migratetype (buddy); + +if (migratetype != buddy_mt + && (!migratetype_is_mergeable (migratetype) || + !migratetype_is_mergeable (buddy_mt))) + goto done_merging; + } +/* * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, * merge with it and move up one order.
@@ -1095,32 +1109,6 @@ static inline void __free_one_page(struct page *page, pfn = combined_pfn; order++; } -if (order <MAX_ORDER -1 ) { -/* If we are here, it means order is >= pageblock_order. -* We want to prevent merge between freepages on pageblock -* without fallbacks and normal pageblock. Without this, -* pageblock isolation could cause incorrect freepage or CMA -* accounting or HIGHATOMIC accounting. -* -* We don't want to hit this code for the more frequent -* low-order merging. -*/ -int buddy_mt; - - buddy_pfn =__find_buddy_pfn (pfn, order); - buddy = page + (buddy_pfn - pfn); - -if (!page_is_buddy (page, buddy, order)) - goto done_merging; - buddy_mt =get_pageblock_migratetype (buddy); - -if (migratetype != buddy_mt - && (!migratetype_is_mergeable (migratetype) || - !migratetype_is_mergeable (buddy_mt))) - goto done_merging; - max_order = order +1 ; - goto continue_merging; - } done_merging:set_buddy_order (page, order);
Signature · 1 line
2.35.1
PATCH v3 2/2 mm: wrap __find_buddy_pfn() with a necessary buddy page validation.
From: Zi Yan <[email protected]>
Whenever the buddy of a page is found from __find_buddy_pfn(), page_is_buddy() should be used to check its validity. Add a helper function find_buddy_page_pfn() to find the buddy page and do the check together.
--- mm/internal.h | 117 ++++++++++++++++++++++++++++++++++---------- mm/page_alloc.c | 52 +++----------------- mm/page_isolation.c | 9 ++-- 3 files changed, 101 insertions(+), 77 deletions(-)
mm/internal.h +90 -27
@@ -211,6 +211,67 @@ struct alloc_context {bool spread_dirty_pages; }; +/* +* This function returns the order of a free page in the buddy system. In +* general, page_zone(page)->lock must be held by the caller to prevent the +* page from being allocated in parallel and returning garbage as the order. +* If a caller does not hold page_zone(page)->lock, it must guarantee that the +* page cannot be allocated or merged in parallel. Alternatively, it must +* handle invalid values gracefully, and use buddy_order_unsafe() below. +*/ +static inline unsigned int buddy_order (struct page *page) +{ +/* PageBuddy() must be checked by the caller */ +return page_private (page); +} + +/* +* Like buddy_order(), but for callers who cannot afford to hold the zone lock. +* PageBuddy() should be checked first by the caller to minimize race window, +* and invalid values must be handled gracefully. +* +* READ_ONCE is used so that if the caller assigns the result into a local +* variable and e.g. tests it for valid range before using, the compiler cannot +* decide to remove the variable and inline the page_private(page) multiple +* times, potentially observing different values in the tests and the actual +* use of the result. +*/ +#define buddy_order_unsafe (page) READ_ONCE(page_private(page)) + +/* +* This function checks whether a page is free && is the buddy +* we can coalesce a page and its buddy if +* (a) the buddy is not in a hole (check before calling!) && +* (b) the buddy is in the buddy system && +* (c) a page and its buddy have the same order && +* (d) a page and its buddy are in the same zone. +* +* For recording whether a page is in the buddy system, we set PageBuddy. +* Setting, clearing, and testing PageBuddy is serialized by zone->lock. +* +* For recording page's order, we use page_private(page). +*/ +static inline bool page_is_buddy (struct page *page,struct page *buddy, +unsigned int order) +{ +if (!page_is_guard (buddy) && !PageBuddy (buddy)) +return false; + +if (buddy_order (buddy) != order) +return false; + +/* +* zone check is done late to avoid uselessly calculating +* zone/node ids for pages that could never merge. +*/ +if (page_zone_id (page) !=page_zone_id (buddy)) +return false; + +VM_BUG_ON_PAGE (page_count (buddy) !=0 , buddy); + +return true; +} +/* * Locate the struct page for both the matching buddy in our * pair (buddy1) and the combined O(n+1) page they form (page).
@@ -234,6 +295,35 @@ __find_buddy_pfn(unsigned long page_pfn, unsigned int order)return page_pfn ^ (1 << order); } +/* +* Find the buddy of @page and validate it. +* @page: The input page +* @pfn: The pfn of the page, it saves a call to page_to_pfn() when the +* function is used in the performance-critical __free_one_page(). +* @order: The order of the page +* @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to +* page_to_pfn(). +* +* The found buddy can be a non PageBuddy, out of @page's zone, or its order is +* not the same as @page. The validation is necessary before use it. +* +* Return: the found buddy page or NULL if not found. +*/ +static inline struct page *find_buddy_page_pfn (struct page *page, +unsigned long pfn,unsigned int order,unsigned long *buddy_pfn) +{ +struct page *buddy; +unsigned long __buddy_pfn =__find_buddy_pfn (pfn, order); + + buddy = page + (__buddy_pfn - pfn); +if (buddy_pfn) + *buddy_pfn = __buddy_pfn; + +if (page_is_buddy (page, buddy, order)) +return buddy; +return NULL ; +} +extern struct page *__pageblock_pfn_to_page (unsigned long start_pfn,unsigned long end_pfn,struct zone *zone);
@@ -336,33 +426,6 @@ isolate_migratepages_range(struct compact_control *cc,int find_suitable_fallback (struct free_area *area,unsigned int order,int migratetype,bool only_stealable,bool *can_steal); -/* -* This function returns the order of a free page in the buddy system. In -* general, page_zone(page)->lock must be held by the caller to prevent the -* page from being allocated in parallel and returning garbage as the order. -* If a caller does not hold page_zone(page)->lock, it must guarantee that the -* page cannot be allocated or merged in parallel. Alternatively, it must -* handle invalid values gracefully, and use buddy_order_unsafe() below. -*/ -static inline unsigned int buddy_order (struct page *page) -{ -/* PageBuddy() must be checked by the caller */ -return page_private (page); -} - -/* -* Like buddy_order(), but for callers who cannot afford to hold the zone lock. -* PageBuddy() should be checked first by the caller to minimize race window, -* and invalid values must be handled gracefully. -* -* READ_ONCE is used so that if the caller assigns the result into a local -* variable and e.g. tests it for valid range before using, the compiler cannot -* decide to remove the variable and inline the page_private(page) multiple -* times, potentially observing different values in the tests and the actual -* use of the result. -*/ -#define buddy_order_unsafe (page) READ_ONCE(page_private(page)) -/* * These three helpers classifies VMAs for virtual memory accounting. */
mm/page_alloc.c +8 -44
76 lines not on this page · raw message
mm/page_isolation.c +3 -6
@@ -70,7 +70,6 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)unsigned long flags, nr_pages;bool isolated_page = false;unsigned int order; -unsigned long pfn, buddy_pfn;struct page *buddy; zone =page_zone (page);
@@ -89,11 +88,9 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)if (PageBuddy (page)) { order =buddy_order (page);if (order >= pageblock_order && order <MAX_ORDER -1 ) { - pfn =page_to_pfn (page); - buddy_pfn =__find_buddy_pfn (pfn, order); - buddy = page + (buddy_pfn - pfn); - -if (!is_migrate_isolate_page (buddy)) { + buddy =find_buddy_page_pfn (page,page_to_pfn (page), + order,NULL ); +if (buddy && !is_migrate_isolate_page (buddy)) { isolated_page = !!__isolate_free_page (page, order);/* * Isolating a free page in an isolated pageblock
Signature · 1 line
2.35.1
Re: [PATCH v3 2/2] mm: wrap __find_buddy_pfn() with a necessary buddy page validation.
Quoted 10 lines · Zi Yan
From: Zi Yan <[email protected]>
Whenever the buddy of a page is found from
__find_buddy_pfn(),page_is_buddy()should be used to check its validity. Add a helper functionfind_buddy_page_pfn()to find the buddy page and do the check together.
Thanks.