convert core hugetlb functions to folios
Subsystems (3)
v3 cover convert core hugetlb functions to folios
============== OVERVIEW ===========================
Now that many hugetlb helper functions that deal with hugetlb specific flags[1] and hugetlb cgroups[2] are converted to folios, higher level allocation, prep, and freeing functions within hugetlb can also be converted to operate in folios.
Patch 1 of this series implements the wrapper functions around setting the compound destructor and compound order for a folio. Besides the user added in patch 1, patch 2 and patch 9 also use these helper functions.
Patches 2-10 convert the higher level hugetlb functions to folios.
============== TESTING ===========================
LTP:
Ran 10 back to back rounds of the LTP hugetlb test suite.
Gigantic Huge Pages:
Test allocation and freeing via hugeadm commands: hugeadm --pool-pages-min 1GB:10 hugeadm --pool-pages-min 1GB:0
Demote:
Demote 1 1GB hugepages to 512 2MB hugepages echo 1 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages echo 1 > /sys/kernel/mm/hugepages/hugepages-1048576kB/demote cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages # 512 cat /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages # 0
Rebased on 10/17/2022 mm-unstable
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/
v1 -> v2:
- fix conflict with "mm,thp,rmap: simplify compound page mapcount handling"
v2 -> v3:
- v2 contained wrong version of patch 1
Sidhartha Kumar (10):
mm: add folio dtor and order setter functions mm/hugetlb: convert destroy_compound_gigantic_page() to folios mm/hugetlb: convert dissolve_free_huge_page() to folios mm/hugetlb: convert remove_hugetlb_page() to folios mm/hugetlb: convert update_and_free_page() to folios mm/hugetlb: convert add_hugetlb_page() to folios and add
hugetlb_cma_folio()
mm/hugetlb: convert enqueue_huge_page() to folios mm/hugetlb: convert free_gigantic_page() to folios mm/hugetlb: convert hugetlb prep functions to folios mm/hugetlb: change hugetlb allocation functions to return a folio
include/linux/mm.h | 16 ++ mm/hugetlb.c | 383 ++++++++++++++++++++++----------------------- 2 files changed, 206 insertions(+), 193 deletions(-)
Signature · 1 line
2.38.1
PATCH v3 1/10 mm: add folio dtor and order setter functions
Add folio equivalents for set_compound_order() and set_compound_page_dtor().
Also remove extra new-lines introduced by mm/hugetlb: convert move_hugetlb_state() to folios and mm/hugetlb_cgroup: convert hugetlb_cgroup_uncharge_page() to folios.
--- include/linux/mm.h | 16 ++++++++++++++++ mm/hugetlb.c | 4 +--- 2 files changed, 17 insertions(+), 3 deletions(-)
include/linux/mm.h +16 -0
@@ -972,6 +972,13 @@ static inline void set_compound_page_dtor(struct page *page, page[1 ].compound_dtor = compound_dtor; } +static inline void folio_set_compound_dtor (struct folio *folio, +enum compound_dtor_id compound_dtor) +{ +VM_BUG_ON_FOLIO (compound_dtor >=NR_COMPOUND_DTORS , folio); + folio->_folio_dtor = compound_dtor; +} +void destroy_large_folio (struct folio *folio);static inline int head_compound_pincount (struct page *head)
@@ -987,6 +994,15 @@ static inline void set_compound_order(struct page *page, unsigned int order)#endif } +static inline void folio_set_compound_order (struct folio *folio, +unsigned int order) +{ + folio->_folio_order = order; +#ifdef CONFIG_64BIT + folio->_folio_nr_pages = order ?1U << order :0 ; +#endif +} +/* Returns the number of pages in this potentially compound page. */ static inline unsigned long compound_nr (struct page *page) {
mm/hugetlb.c +1 -3
@@ -1780,7 +1780,7 @@ static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio) {hugetlb_vmemmap_optimize (h, &folio->page);INIT_LIST_HEAD (&folio->lru); - folio->_folio_dtor =HUGETLB_PAGE_DTOR ; +folio_set_compound_dtor (folio,HUGETLB_PAGE_DTOR );hugetlb_set_folio_subpool (folio,NULL );set_hugetlb_cgroup (folio,NULL );set_hugetlb_cgroup_rsvd (folio,NULL );
@@ -2936,7 +2936,6 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,* a reservation exists for the allocation. */ page =dequeue_huge_page_vma (h, vma, addr, avoid_reserve, gbl_chg); -if (!page) {spin_unlock_irq (&hugetlb_lock); page =alloc_buddy_huge_page_with_mpol (h, vma, addr);
@@ -7349,7 +7348,6 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reint old_nid =folio_nid (old_folio);int new_nid =folio_nid (new_folio); -folio_set_hugetlb_temporary (old_folio);folio_clear_hugetlb_temporary (new_folio);
Signature · 1 line
2.38.1
PATCH v3 2/10 mm/hugetlb: convert destroy_compound_gigantic_page() to folios
In reply to Sidhartha KumarConvert page operations within __destroy_compound_gigantic_page() to the corresponding folio operations.
--- mm/hugetlb.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-)
mm/hugetlb.c +21 -22
@@ -1325,43 +1325,40 @@ static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed) nr_nodes--)/* used to demote non-gigantic_huge pages as well */ -static void __destroy_compound_gigantic_page (struct page *page, +static void __destroy_compound_gigantic_folio (struct folio *folio,unsigned int order,bool demote) {int i;int nr_pages =1 << order;struct page *p; -atomic_set (compound_mapcount_ptr (page),0 ); -atomic_set (subpages_mapcount_ptr (page),0 ); -atomic_set (compound_pincount_ptr (page),0 ); +atomic_set (folio_mapcount_ptr (folio),0 ); +atomic_set (folio_subpages_mapcount_ptr (folio),0 ); +atomic_set (folio_pincount_ptr (folio),0 );for (i =1 ; i < nr_pages; i++) { - p =nth_page (page, i); + p =folio_page (folio, i); p->mapping =NULL ;clear_compound_head (p);if (!demote)set_page_refcounted (p); } -set_compound_order (page,0 ); -#ifdef CONFIG_64BIT - page[1 ].compound_nr =0 ; -#endif -__ClearPageHead (page); +folio_set_compound_order (folio,0 ); +folio_clear_head (folio); } -static void destroy_compound_hugetlb_page_for_demote (struct page *page, +static void destroy_compound_hugetlb_folio_for_demote (struct folio *folio,unsigned int order) { -__destroy_compound_gigantic_page (page, order, true); +__destroy_compound_gigantic_folio (folio, order, true); }#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE -static void destroy_compound_gigantic_page (struct page *page, +static void destroy_compound_gigantic_folio (struct folio *folio,unsigned int order) { -__destroy_compound_gigantic_page (page, order, false); +__destroy_compound_gigantic_folio (folio, order, false); }static void free_gigantic_page (struct page *page,unsigned int order)
@@ -1430,7 +1427,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, return NULL; }static inline void free_gigantic_page (struct page *page,unsigned int order) { } -static inline void destroy_compound_gigantic_page (struct page *page, +static inline void destroy_compound_gigantic_folio (struct folio *folio,unsigned int order) { }#endif
@@ -1477,8 +1474,8 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,* * For gigantic pages set the destructor to the null dtor. This * destructor will never be called. Before freeing the gigantic -* page destroy_compound_gigantic_page will turn the compound page -* into a simple group of pages. After this the destructor does not +* page destroy_compound_gigantic_folio will turn the folio into a +* simple group of pages. After this the destructor does not * apply. * * This handles the case where more than one ref is held when and
@@ -1559,6 +1556,7 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,static void __update_and_free_page (struct hstate *h,struct page *page) {int i; +struct folio *folio =page_folio (page);struct page *subpage;if (hstate_is_gigantic (h) && !gigantic_page_runtime_supported ())
@@ -1587,8 +1585,8 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* Move PageHWPoison flag from head page to the raw error pages, * which makes any healthy subpages reusable. */ -if (unlikely (PageHWPoison (page))) -hugetlb_clear_page_hwpoison (page); +if (unlikely (folio_test_hwpoison (folio))) +hugetlb_clear_page_hwpoison (&folio->page);for (i =0 ; i <pages_per_huge_page (h); i++) { subpage =nth_page (page, i);
@@ -1604,7 +1602,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)*/ if (hstate_is_gigantic (h) ||hugetlb_cma_page (page,huge_page_order (h))) { -destroy_compound_gigantic_page (page,huge_page_order (h)); +destroy_compound_gigantic_folio (folio,huge_page_order (h));free_gigantic_page (page,huge_page_order (h)); }else {__free_pages (page,huge_page_order (h));
@@ -3435,6 +3433,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) {int i, nid =page_to_nid (page);struct hstate *target_hstate; +struct folio *folio =page_folio (page);struct page *subpage;int rc =0 ;
@@ -3453,10 +3452,10 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) }/* -* Use destroy_compound_hugetlb_page_for_demote for all huge page +* Use destroy_compound_hugetlb_folio_for_demote for all huge page * sizes as it will not ref count pages. */ -destroy_compound_hugetlb_page_for_demote (page,huge_page_order (h)); +destroy_compound_hugetlb_folio_for_demote (folio,huge_page_order (h));/* * Taking target hstate mutex synchronizes with set_max_huge_pages.
Signature · 1 line
2.38.1
PATCH v3 3/10 mm/hugetlb: convert dissolve_free_huge_page() to folios
In reply to Sidhartha KumarRemoves compound_head() call by using a folio rather than a head page.
--- mm/hugetlb.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
mm/hugetlb.c +10 -10
@@ -2126,21 +2126,21 @@ static struct page *remove_pool_huge_page(struct hstate *h,int dissolve_free_huge_page (struct page *page) {int rc = -EBUSY ; +struct folio *folio =page_folio (page); retry:/* Not to disrupt normal path by vainly holding hugetlb_lock */ -if (!PageHuge (page)) +if (!folio_test_hugetlb (folio))return 0 ;spin_lock_irq (&hugetlb_lock); -if (!PageHuge (page)) { +if (!folio_test_hugetlb (folio)) { rc =0 ; goto out; } -if (!page_count (page)) { -struct page *head =compound_head (page); -struct hstate *h =page_hstate (head); +if (!folio_ref_count (folio)) { +struct hstate *h =folio_hstate (folio);if (!available_huge_pages (h)) goto out;
@@ -2148,7 +2148,7 @@ int dissolve_free_huge_page(struct page *page)* We should make sure that the page is already on the free list * when it is dissolved. */ -if (unlikely (!HPageFreed (head))) { +if (unlikely (!folio_test_hugetlb_freed (folio))) {spin_unlock_irq (&hugetlb_lock);cond_resched ();
@@ -2163,7 +2163,7 @@ int dissolve_free_huge_page(struct page *page) goto retry; } -remove_hugetlb_page (h, head, false); +remove_hugetlb_page (h, &folio->page, false); h->max_huge_pages--;spin_unlock_irq (&hugetlb_lock);
@@ -2175,12 +2175,12 @@ int dissolve_free_huge_page(struct page *page)* Attempt to allocate vmemmmap here so that we can take * appropriate action on failure. */ - rc =hugetlb_vmemmap_restore (h, head); + rc =hugetlb_vmemmap_restore (h, &folio->page);if (!rc) { -update_and_free_page (h, head, false); +update_and_free_page (h, &folio->page, false); }else {spin_lock_irq (&hugetlb_lock); -add_hugetlb_page (h, head, false); +add_hugetlb_page (h, &folio->page, false); h->max_huge_pages++;spin_unlock_irq (&hugetlb_lock); }
Signature · 1 line
2.38.1
PATCH v3 4/10 mm/hugetlb: convert remove_hugetlb_page() to folios
In reply to Sidhartha KumarRemoves page_folio() call by converting callers to directly pass a folio into __remove_hugetlb_page().
--- mm/hugetlb.c | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-)
mm/hugetlb.c +25 -23
@@ -1432,19 +1432,18 @@ static inline void destroy_compound_gigantic_folio(struct folio *folio,#endif /* -* Remove hugetlb page from lists, and update dtor so that page appears +* Remove hugetlb folio from lists, and update dtor so that the folio appears * as just a compound page. * -* A reference is held on the page, except in the case of demote. +* A reference is held on the folio, except in the case of demote. * * Must be called with hugetlb lock held. */ -static void __remove_hugetlb_page (struct hstate *h,struct page *page, +static void __remove_hugetlb_folio (struct hstate *h,struct folio *folio,bool adjust_surplus,bool demote) { -int nid =page_to_nid (page); -struct folio *folio =page_folio (page); +int nid =folio_nid (folio);VM_BUG_ON_FOLIO (hugetlb_cgroup_from_folio (folio), folio);VM_BUG_ON_FOLIO (hugetlb_cgroup_from_folio_rsvd (folio), folio);
@@ -1453,9 +1452,9 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,if (hstate_is_gigantic (h) && !gigantic_page_runtime_supported ())return ; -list_del (&page->lru); +list_del (&folio->lru); -if (HPageFreed (page)) { +if (folio_test_hugetlb_freed (folio)) { h->free_huge_pages--; h->free_huge_pages_node[nid]--; }
@@ -1485,26 +1484,26 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,* be turned into a page of smaller size. */ if (!demote) -set_page_refcounted (page); +folio_ref_unfreeze (folio,1 );if (hstate_is_gigantic (h)) -set_compound_page_dtor (page,NULL_COMPOUND_DTOR ); +folio_set_compound_dtor (folio,NULL_COMPOUND_DTOR );else -set_compound_page_dtor (page,COMPOUND_PAGE_DTOR ); +folio_set_compound_dtor (folio,COMPOUND_PAGE_DTOR ); h->nr_huge_pages--; h->nr_huge_pages_node[nid]--; } -static void remove_hugetlb_page (struct hstate *h,struct page *page, +static void remove_hugetlb_folio (struct hstate *h,struct folio *folio,bool adjust_surplus) { -__remove_hugetlb_page (h, page, adjust_surplus, false); +__remove_hugetlb_folio (h, folio, adjust_surplus, false); } -static void remove_hugetlb_page_for_demote (struct hstate *h,struct page *page, +static void remove_hugetlb_folio_for_demote (struct hstate *h,struct folio *folio,bool adjust_surplus) { -__remove_hugetlb_page (h, page, adjust_surplus, true); +__remove_hugetlb_folio (h, folio, adjust_surplus, true); } static void add_hugetlb_page(struct hstate *h, struct page *page,
@@ -1639,8 +1638,9 @@ static void free_hpage_workfn(struct work_struct *work)/* * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate() * is going to trigger because a previous call to -* remove_hugetlb_page() will set_compound_page_dtor(page, -* NULL_COMPOUND_DTOR), so do not use page_hstate() directly. +* remove_hugetlb_folio() will call folio_set_compound_dtor +* (folio, NULL_COMPOUND_DTOR), so do not use page_hstate() +* directly. */ h =size_to_hstate (page_size (page));
@@ -1749,12 +1749,12 @@ void free_huge_page(struct page *page) h->resv_huge_pages++;if (folio_test_hugetlb_temporary (folio)) { -remove_hugetlb_page (h, page, false); +remove_hugetlb_folio (h, folio, false);spin_unlock_irqrestore (&hugetlb_lock, flags);update_and_free_page (h, page, true); }else if (h->surplus_huge_pages_node[nid]) {/* remove the page from active list */ -remove_hugetlb_page (h, page, true); +remove_hugetlb_folio (h, folio, true);spin_unlock_irqrestore (&hugetlb_lock, flags);update_and_free_page (h, page, true); }else {
@@ -2090,6 +2090,7 @@ static struct page *remove_pool_huge_page(struct hstate *h, {int nr_nodes, node;struct page *page =NULL ; +struct folio *folio;lockdep_assert_held (&hugetlb_lock);for_each_node_mask_to_free (h, nr_nodes, node, nodes_allowed) {
@@ -2101,7 +2102,8 @@ static struct page *remove_pool_huge_page(struct hstate *h, !list_empty(&h->hugepage_freelists[node])) { page =list_entry (h->hugepage_freelists[node].next, struct page, lru); -remove_hugetlb_page (h, page, acct_surplus); + folio =page_folio (page); +remove_hugetlb_folio (h, folio, acct_surplus);break ; } }
@@ -2163,7 +2165,7 @@ int dissolve_free_huge_page(struct page *page) goto retry; } -remove_hugetlb_page (h, &folio->page, false); +remove_hugetlb_folio (h, folio, false); h->max_huge_pages--;spin_unlock_irq (&hugetlb_lock);
@@ -2801,7 +2803,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,* and enqueue_huge_page() for new_page. The counters will remain * stable since this happens under the lock. */ -remove_hugetlb_page (h, old_page, false); +remove_hugetlb_folio (h, old_folio, false);/* * Ref count on new page is already zero as it was dropped
@@ -3228,7 +3230,7 @@ static void try_to_free_low(struct hstate *h, unsigned long count, goto out;if (PageHighMem (page))continue ; -remove_hugetlb_page (h, page, false); +remove_hugetlb_folio (h,page_folio (page), false);list_add (&page->lru, &page_list); } }
@@ -3439,7 +3441,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) target_hstate =size_to_hstate (PAGE_SIZE << h->demote_order); -remove_hugetlb_page_for_demote (h, page, false); +remove_hugetlb_folio_for_demote (h, folio, false);spin_unlock_irq (&hugetlb_lock); rc =hugetlb_vmemmap_restore (h, page);
Signature · 1 line
2.38.1
PATCH v3 5/10 mm/hugetlb: convert update_and_free_page() to folios
In reply to Sidhartha KumarMake more progress on converting the free_huge_page() destructor to operate on folios by converting update_and_free_page() to folios.
--- mm/hugetlb.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-)
mm/hugetlb.c +16 -14
@@ -1478,7 +1478,7 @@ static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,* apply. * * This handles the case where more than one ref is held when and -* after update_and_free_page is called. +* after update_and_free_hugetlb_folio is called. * * In the case of demote we do not ref count the page as it will soon * be turned into a page of smaller size.
@@ -1609,7 +1609,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page) }/* -* As update_and_free_page() can be called under any context, so we cannot +* As update_and_free_hugetlb_folio() can be called under any context, so we cannot * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate * the vmemmap pages.
@@ -1657,11 +1657,11 @@ static inline void flush_free_hpage_work(struct hstate *h)flush_work (&free_hpage_work); } -static void update_and_free_page (struct hstate *h,struct page *page, +static void update_and_free_hugetlb_folio (struct hstate *h,struct folio *folio,bool atomic) { -if (!HPageVmemmapOptimized (page) || !atomic) { -__update_and_free_page (h, page); +if (!folio_test_hugetlb_vmemmap_optimized (folio) || !atomic) { +__update_and_free_page (h, &folio->page);return ; }
@@ -1672,16 +1672,18 @@ static void update_and_free_page(struct hstate *h, struct page *page,* empty. Otherwise, schedule_work() had been called but the workfn * hasn't retrieved the list yet. */ -if (llist_add ((struct llist_node *)&page->mapping, &hpage_freelist)) +if (llist_add ((struct llist_node *)&folio->mapping, &hpage_freelist))schedule_work (&free_hpage_work); }static void update_and_free_pages_bulk (struct hstate *h,struct list_head *list) {struct page *page, *t_page; +struct folio *folio;list_for_each_entry_safe (page, t_page, list, lru) { -update_and_free_page (h, page, false); + folio =page_folio (page); +update_and_free_hugetlb_folio (h, folio, false);cond_resched (); } }
@@ -1751,12 +1753,12 @@ void free_huge_page(struct page *page)if (folio_test_hugetlb_temporary (folio)) {remove_hugetlb_folio (h, folio, false);spin_unlock_irqrestore (&hugetlb_lock, flags); -update_and_free_page (h, page, true); +update_and_free_hugetlb_folio (h, folio, true); }else if (h->surplus_huge_pages_node[nid]) {/* remove the page from active list */ remove_hugetlb_folio (h, folio, true);spin_unlock_irqrestore (&hugetlb_lock, flags); -update_and_free_page (h, page, true); +update_and_free_hugetlb_folio (h, folio, true); }else {arch_clear_hugepage_flags (page);enqueue_huge_page (h, page);
@@ -2170,8 +2172,8 @@ int dissolve_free_huge_page(struct page *page)spin_unlock_irq (&hugetlb_lock);/* -* Normally update_and_free_page will allocate required vmemmmap -* before freeing the page. update_and_free_page will fail to +* Normally update_and_free_hugtlb_folio will allocate required vmemmmap +* before freeing the page. update_and_free_hugtlb_folio will fail to * free the page if it can not allocate required vmemmap. We * need to adjust max_huge_pages if the page is not freed. * Attempt to allocate vmemmmap here so that we can take
@@ -2179,7 +2181,7 @@ int dissolve_free_huge_page(struct page *page)*/ rc =hugetlb_vmemmap_restore (h, &folio->page);if (!rc) { -update_and_free_page (h, &folio->page, false); +update_and_free_hugetlb_folio (h, folio, false); }else {spin_lock_irq (&hugetlb_lock);add_hugetlb_page (h, &folio->page, false);
@@ -2816,7 +2818,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,* Pages have been replaced, we can safely free the old one. */ spin_unlock_irq (&hugetlb_lock); -update_and_free_page (h, old_page, false); +update_and_free_hugetlb_folio (h, old_folio, false); }return ret;
@@ -2825,7 +2827,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,spin_unlock_irq (&hugetlb_lock);/* Page has a zero ref count, but needs a ref to be freed */ folio_ref_unfreeze (new_folio,1 ); -update_and_free_page (h, new_page, false); +update_and_free_hugetlb_folio (h, new_folio, false);return ret; }
Signature · 1 line
2.38.1
PATCH v3 6/10 mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
In reply to Sidhartha KumarConvert add_hugetlb_page() to take in a folio, also convert hugetlb_cma_page() to take in a folio.
--- mm/hugetlb.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-)
mm/hugetlb.c +20 -20
@@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];#ifdef CONFIG_CMA static struct cma *hugetlb_cma[MAX_NUMNODES ];static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES ]__initdata ; -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) { -return cma_pages_valid (hugetlb_cma[page_to_nid (page)], page, +return cma_pages_valid (hugetlb_cma[folio_nid (folio)], &folio->page,1 << order); }#else -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) {return false; }
@@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli__remove_hugetlb_folio (h, folio, adjust_surplus, true); } -static void add_hugetlb_page (struct hstate *h,struct page *page, +static void add_hugetlb_folio (struct hstate *h,struct folio *folio,bool adjust_surplus) {int zeroed; -int nid =page_to_nid (page); +int nid =folio_nid (folio); -VM_BUG_ON_PAGE (!HPageVmemmapOptimized (page), page); +VM_BUG_ON_FOLIO (!folio_test_hugetlb_vmemmap_optimized (folio), folio);lockdep_assert_held (&hugetlb_lock); -INIT_LIST_HEAD (&page->lru); +INIT_LIST_HEAD (&folio->lru); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++;
@@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page, h->surplus_huge_pages_node[nid]++; } -set_compound_page_dtor (page,HUGETLB_PAGE_DTOR ); -set_page_private (page,0 ); +folio_set_compound_dtor (folio,HUGETLB_PAGE_DTOR ); +folio_change_private (folio,0 );/* * We have to set HPageVmemmapOptimized again as above -* set_page_private(page, 0) cleared it. +* folio_change_private(folio, 0) cleared it. */ -SetHPageVmemmapOptimized (page); +folio_set_hugetlb_vmemmap_optimized (folio);/* -* This page is about to be managed by the hugetlb allocator and +* This folio is about to be managed by the hugetlb allocator and * should have no users. Drop our reference, and check for others * just in case. */ - zeroed =put_page_testzero (page); -if (!zeroed) + zeroed =folio_put_testzero (folio); +if (unlikely (!zeroed))/* * It is VERY unlikely soneone else has taken a ref on * the page. In this case, we simply return as the
@@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,*/ return ; -arch_clear_hugepage_flags (page); -enqueue_huge_page (h, page); +arch_clear_hugepage_flags (&folio->page); +enqueue_huge_page (h, &folio->page); }static void __update_and_free_page (struct hstate *h,struct page *page)
@@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* page and put the page back on the hugetlb free list and treat * as a surplus page. */ -add_hugetlb_page (h, page, true); +add_hugetlb_folio (h,page_folio (page), true);spin_unlock_irq (&hugetlb_lock);return ; }
@@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* need to be given back to CMA in free_gigantic_page. */ if (hstate_is_gigantic (h) || -hugetlb_cma_page (page,huge_page_order (h))) { +hugetlb_cma_folio (folio,huge_page_order (h))) {destroy_compound_gigantic_folio (folio,huge_page_order (h));free_gigantic_page (page,huge_page_order (h)); }else {
@@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)update_and_free_hugetlb_folio (h, folio, false); }else {spin_lock_irq (&hugetlb_lock); -add_hugetlb_page (h, &folio->page, false); +add_hugetlb_folio (h, folio, false); h->max_huge_pages++;spin_unlock_irq (&hugetlb_lock); }
@@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)/* Allocation of vmemmmap failed, we can not demote page */ spin_lock_irq (&hugetlb_lock);set_page_refcounted (page); -add_hugetlb_page (h, page, false); +add_hugetlb_folio (h,page_folio (page), false);return rc; }
Signature · 1 line
2.38.1
Re: [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
Hi,
Though it is already merged, it is just comment thing.
Quoted 52 lines · Sidhartha Kumar
Convert
add_hugetlb_page()to take in a folio, also converthugetlb_cma_page()to take in a folio.Signed-off-by Sidhartha Kumar <[email protected]>--- mm/hugetlb.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-)mm/hugetlb.c +7
-7@@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];#ifdef CONFIG_CMA static struct cma *hugetlb_cma[MAX_NUMNODES ];static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES ]__initdata ; -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) { -return cma_pages_valid (hugetlb_cma[page_to_nid (page)], page, +return cma_pages_valid (hugetlb_cma[folio_nid (folio)], &folio->page,1 << order); }#else -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) {return false; }@@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli__remove_hugetlb_folio (h, folio, adjust_surplus, true); } -static void add_hugetlb_page (struct hstate *h,struct page *page, +static void add_hugetlb_folio (struct hstate *h,struct folio *folio,bool adjust_surplus) {int zeroed; -int nid =page_to_nid (page); +int nid =folio_nid (folio); -VM_BUG_ON_PAGE (!HPageVmemmapOptimized (page), page); +VM_BUG_ON_FOLIO (!folio_test_hugetlb_vmemmap_optimized (folio), folio);lockdep_assert_held (&hugetlb_lock); -INIT_LIST_HEAD (&page->lru); +INIT_LIST_HEAD (&folio->lru); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++;
mm/hugetlb.c static void add_hugetlb_page(struct hstate *h, struct page *page, · line 1531
Quoted 9 lines
mm/hugetlb.c +2
-2@@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page, h->surplus_huge_pages_node[nid]++; } -set_compound_page_dtor (page,HUGETLB_PAGE_DTOR ); -set_page_private (page,0 ); +folio_set_compound_dtor (folio,HUGETLB_PAGE_DTOR ); +folio_change_private (folio,0 );/* * We have to set HPageVmemmapOptimized again as above
^
This can be changed to folio version of itself.
Quoted 69 lines
mm/hugetlb.c +11
-11-* set_page_private(page, 0) cleared it. +* folio_change_private(folio, 0) cleared it. */ -SetHPageVmemmapOptimized (page); +folio_set_hugetlb_vmemmap_optimized (folio);/* -* This page is about to be managed by the hugetlb allocator and +* This folio is about to be managed by the hugetlb allocator and * should have no users. Drop our reference, and check for others * just in case. */ - zeroed =put_page_testzero (page); -if (!zeroed) + zeroed =folio_put_testzero (folio); +if (unlikely (!zeroed))/* * It is VERY unlikely soneone else has taken a ref on * the page. In this case, we simply return as the @@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,*/ return ; -arch_clear_hugepage_flags (page); -enqueue_huge_page (h, page); +arch_clear_hugepage_flags (&folio->page); +enqueue_huge_page (h, &folio->page); }static void __update_and_free_page (struct hstate *h,struct page *page)@@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* page and put the page back on the hugetlb free list and treat * as a surplus page. */ -add_hugetlb_page (h, page, true); +add_hugetlb_folio (h,page_folio (page), true);spin_unlock_irq (&hugetlb_lock);return ; }@@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* need to be given back to CMA in free_gigantic_page. */ if (hstate_is_gigantic (h) || -hugetlb_cma_page (page,huge_page_order (h))) { +hugetlb_cma_folio (folio,huge_page_order (h))) {destroy_compound_gigantic_folio (folio,huge_page_order (h));free_gigantic_page (page,huge_page_order (h)); }else {@@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)update_and_free_hugetlb_folio (h, folio, false); }else {spin_lock_irq (&hugetlb_lock); -add_hugetlb_page (h, &folio->page, false); +add_hugetlb_folio (h, folio, false); h->max_huge_pages++;spin_unlock_irq (&hugetlb_lock); }@@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)/* Allocation of vmemmmap failed, we can not demote page */ spin_lock_irq (&hugetlb_lock);set_page_refcounted (page); -add_hugetlb_page (h, page, false); +add_hugetlb_folio (h,page_folio (page), false);return rc; }Signature · 1 line
2.38.1
Re: [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
Quoted 2 lines · Tarun Sahu
Hi,
Though it is already merged, it is just comment thing.
This series will need another version based on other feedback that I got. I'll be sure to change the comment in the next version.
Thanks,
Sidhartha Kumar
Quoted 134 lines · Sidhartha Kumar
Quoted 52 lines
Convert
add_hugetlb_page()to take in a folio, also converthugetlb_cma_page()to take in a folio.Signed-off-by Sidhartha Kumar <[email protected]>--- mm/hugetlb.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-)mm/hugetlb.c +7
-7@@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];#ifdef CONFIG_CMA static struct cma *hugetlb_cma[MAX_NUMNODES ];static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES ]__initdata ; -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) { -return cma_pages_valid (hugetlb_cma[page_to_nid (page)], page, +return cma_pages_valid (hugetlb_cma[folio_nid (folio)], &folio->page,1 << order); }#else -static bool hugetlb_cma_page (struct page *page,unsigned int order) +static bool hugetlb_cma_folio (struct folio *folio,unsigned int order) {return false; }@@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli__remove_hugetlb_folio (h, folio, adjust_surplus, true); } -static void add_hugetlb_page (struct hstate *h,struct page *page, +static void add_hugetlb_folio (struct hstate *h,struct folio *folio,bool adjust_surplus) {int zeroed; -int nid =page_to_nid (page); +int nid =folio_nid (folio); -VM_BUG_ON_PAGE (!HPageVmemmapOptimized (page), page); +VM_BUG_ON_FOLIO (!folio_test_hugetlb_vmemmap_optimized (folio), folio);lockdep_assert_held (&hugetlb_lock); -INIT_LIST_HEAD (&page->lru); +INIT_LIST_HEAD (&folio->lru); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++;mm/hugetlb.c static void add_hugetlb_page(struct hstate *h, struct page *page, · line 1531
Quoted 9 lines
mm/hugetlb.c +2
-2@@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page, h->surplus_huge_pages_node[nid]++; } -set_compound_page_dtor (page,HUGETLB_PAGE_DTOR ); -set_page_private (page,0 ); +folio_set_compound_dtor (folio,HUGETLB_PAGE_DTOR ); +folio_change_private (folio,0 );/* * We have to set HPageVmemmapOptimized again as above ^This can be changed to folio version of itself.
Quoted 69 lines
mm/hugetlb.c +11
-11-* set_page_private(page, 0) cleared it. +* folio_change_private(folio, 0) cleared it. */ -SetHPageVmemmapOptimized (page); +folio_set_hugetlb_vmemmap_optimized (folio);/* -* This page is about to be managed by the hugetlb allocator and +* This folio is about to be managed by the hugetlb allocator and * should have no users. Drop our reference, and check for others * just in case. */ - zeroed =put_page_testzero (page); -if (!zeroed) + zeroed =folio_put_testzero (folio); +if (unlikely (!zeroed))/* * It is VERY unlikely soneone else has taken a ref on * the page. In this case, we simply return as the @@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,*/ return ; -arch_clear_hugepage_flags (page); -enqueue_huge_page (h, page); +arch_clear_hugepage_flags (&folio->page); +enqueue_huge_page (h, &folio->page); }static void __update_and_free_page (struct hstate *h,struct page *page)@@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* page and put the page back on the hugetlb free list and treat * as a surplus page. */ -add_hugetlb_page (h, page, true); +add_hugetlb_folio (h,page_folio (page), true);spin_unlock_irq (&hugetlb_lock);return ; }@@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* need to be given back to CMA in free_gigantic_page. */ if (hstate_is_gigantic (h) || -hugetlb_cma_page (page,huge_page_order (h))) { +hugetlb_cma_folio (folio,huge_page_order (h))) {destroy_compound_gigantic_folio (folio,huge_page_order (h));free_gigantic_page (page,huge_page_order (h)); }else {@@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)update_and_free_hugetlb_folio (h, folio, false); }else {spin_lock_irq (&hugetlb_lock); -add_hugetlb_page (h, &folio->page, false); +add_hugetlb_folio (h, folio, false); h->max_huge_pages++;spin_unlock_irq (&hugetlb_lock); }@@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)/* Allocation of vmemmmap failed, we can not demote page */ spin_lock_irq (&hugetlb_lock);set_page_refcounted (page); -add_hugetlb_page (h, page, false); +add_hugetlb_folio (h,page_folio (page), false);return rc; }Signature · 1 line
2.38.1
PATCH v3 7/10 mm/hugetlb: convert enqueue_huge_page() to folios
In reply to Sidhartha KumarConvert callers of enqueue_huge_page() to pass in a folio, function is renamed to enqueue_hugetlb_folio().
--- mm/hugetlb.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
mm/hugetlb.c +11 -11
@@ -1127,17 +1127,17 @@ static bool vma_has_reserves(struct vm_area_struct *vma, long chg)return false; } -static void enqueue_huge_page (struct hstate *h,struct page *page) +static void enqueue_hugetlb_folio (struct hstate *h,struct folio *folio) { -int nid =page_to_nid (page); +int nid =folio_nid (folio);lockdep_assert_held (&hugetlb_lock); -VM_BUG_ON_PAGE (page_count (page), page); +VM_BUG_ON_FOLIO (folio_ref_count (folio), folio); -list_move (&page->lru, &h->hugepage_freelists[nid]); +list_move (&folio->lru, &h->hugepage_freelists[nid]); h->free_huge_pages++; h->free_huge_pages_node[nid]++; -SetHPageFreed (page); +folio_set_hugetlb_freed (folio); }static struct page *dequeue_huge_page_node_exact (struct hstate *h,int nid)
@@ -1549,7 +1549,7 @@ static void add_hugetlb_folio(struct hstate *h, struct folio *folio,return ;arch_clear_hugepage_flags (&folio->page); -enqueue_huge_page (h, &folio->page); +enqueue_hugetlb_folio (h, folio); }static void __update_and_free_page (struct hstate *h,struct page *page)
@@ -1761,7 +1761,7 @@ void free_huge_page(struct page *page)update_and_free_hugetlb_folio (h, folio, true); }else {arch_clear_hugepage_flags (page); -enqueue_huge_page (h, page); +enqueue_hugetlb_folio (h, folio);spin_unlock_irqrestore (&hugetlb_lock, flags); } }
@@ -2436,7 +2436,7 @@ static int gather_surplus_pages(struct hstate *h, long delta)if ((--needed) <0 )break ;/* Add the page to the hugetlb allocator */ -enqueue_huge_page (h, page); +enqueue_hugetlb_folio (h,page_folio (page)); } free:spin_unlock_irq (&hugetlb_lock);
@@ -2802,8 +2802,8 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,* Ok, old_page is still a genuine free hugepage. Remove it from * the freelist and decrease the counters. These will be * incremented again when calling __prep_account_new_huge_page() -* and enqueue_huge_page() for new_page. The counters will remain -* stable since this happens under the lock. +* and enqueue_hugetlb_folio() for new_folio. The counters will +* remain stable since this happens under the lock. */ remove_hugetlb_folio (h, old_folio, false);
@@ -2812,7 +2812,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,* earlier. It can be directly added to the pool free list. */ __prep_account_new_huge_page (h, nid); -enqueue_huge_page (h, new_page); +enqueue_hugetlb_folio (h, new_folio);/* * Pages have been replaced, we can safely free the old one.
Signature · 1 line
2.38.1
PATCH v3 8/10 mm/hugetlb: convert free_gigantic_page() to folios
In reply to Sidhartha KumarConvert callers of free_gigantic_page() to use folios, function is then renamed to free_gigantic_folio().
--- mm/hugetlb.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-)
mm/hugetlb.c +17 -12
@@ -1361,18 +1361,20 @@ static void destroy_compound_gigantic_folio(struct folio *folio,__destroy_compound_gigantic_folio (folio, order, false); } -static void free_gigantic_page (struct page *page,unsigned int order) +static void free_gigantic_folio (struct folio *folio,unsigned int order) {/* * If the page isn't allocated using the cma allocator, * cma_release() returns false. */ #ifdef CONFIG_CMA -if (cma_release (hugetlb_cma[page_to_nid (page)], page,1 << order)) +int nid =folio_nid (folio); + +if (cma_release (hugetlb_cma[nid], &folio->page,1 << order))return ;#endif -free_contig_range (page_to_pfn (page),1 << order); +free_contig_range (folio_pfn (folio),1 << order); }#ifdef CONFIG_CONTIG_ALLOC
@@ -1426,7 +1428,8 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, {return NULL ; } -static inline void free_gigantic_page (struct page *page,unsigned int order) { } +static inline void free_gigantic_folio (struct folio *folio, +unsigned int order) { }static inline void destroy_compound_gigantic_folio (struct folio *folio,unsigned int order) { }#endif
@@ -1565,7 +1568,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* If we don't know which subpages are hwpoisoned, we can't free * the hugepage, so it's leaked intentionally. */ -if (HPageRawHwpUnreliable (page)) +if (folio_test_hugetlb_raw_hwp_unreliable (folio))return ;if (hugetlb_vmemmap_restore (h, page)) {
@@ -1575,7 +1578,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)* page and put the page back on the hugetlb free list and treat * as a surplus page. */ -add_hugetlb_folio (h,page_folio (page), true); +add_hugetlb_folio (h, folio, true);spin_unlock_irq (&hugetlb_lock);return ; }
@@ -1588,7 +1591,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)hugetlb_clear_page_hwpoison (&folio->page);for (i =0 ; i <pages_per_huge_page (h); i++) { - subpage =nth_page (page, i); + subpage =folio_page (folio, i); subpage->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | 1 << PG_dirty | 1 << PG_active | 1 << PG_private |
@@ -1597,12 +1600,12 @@ static void __update_and_free_page(struct hstate *h, struct page *page)/* * Non-gigantic pages demoted from CMA allocated gigantic pages -* need to be given back to CMA in free_gigantic_page. +* need to be given back to CMA in free_gigantic_folio. */ if (hstate_is_gigantic (h) ||hugetlb_cma_folio (folio,huge_page_order (h))) {destroy_compound_gigantic_folio (folio,huge_page_order (h)); -free_gigantic_page (page,huge_page_order (h)); +free_gigantic_folio (folio,huge_page_order (h)); }else {__free_pages (page,huge_page_order (h)); }
@@ -2023,6 +2026,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h, nodemask_t *node_alloc_noretry) {struct page *page; +struct folio *folio;bool retry = false; retry:
@@ -2033,14 +2037,14 @@ static struct page *alloc_fresh_huge_page(struct hstate *h, nid, nmask, node_alloc_noretry);if (!page)return NULL ; - + folio =page_folio (page);if (hstate_is_gigantic (h)) {if (!prep_compound_gigantic_page (page,huge_page_order (h))) {/* * Rare failure to convert pages to compound page. * Free pages and try again - ONCE! */ -free_gigantic_page (page,huge_page_order (h)); +free_gigantic_folio (folio,huge_page_order (h));if (!retry) { retry = true; goto retry;
@@ -3048,6 +3052,7 @@ static void __init gather_bootmem_prealloc(void)list_for_each_entry (m, &huge_boot_pages, list) {struct page *page =virt_to_page (m); +struct folio *folio =page_folio (page);struct hstate *h = m->hstate;VM_BUG_ON (!hstate_is_gigantic (h));
@@ -3058,7 +3063,7 @@ static void __init gather_bootmem_prealloc(void)free_huge_page (page);/* add to the hugepage allocator */ }else {/* VERY unlikely inflated ref count on a tail page */ -free_gigantic_page (page,huge_page_order (h)); +free_gigantic_folio (folio,huge_page_order (h)); }/*
Signature · 1 line
2.38.1
PATCH v3 9/10 mm/hugetlb: convert hugetlb prep functions to folios
In reply to Sidhartha KumarConvert prep_new_huge_page() and __prep_compound_gigantic_page() to folios.
--- mm/hugetlb.c | 61 +++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 32 deletions(-)
mm/hugetlb.c +29 -32
@@ -1789,28 +1789,26 @@ static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)set_hugetlb_cgroup_rsvd (folio,NULL ); } -static void prep_new_huge_page (struct hstate *h,struct page *page,int nid) +static void prep_new_hugetlb_folio (struct hstate *h,struct folio *folio,int nid) { -struct folio *folio =page_folio (page); -__prep_new_hugetlb_folio (h, folio);spin_lock_irq (&hugetlb_lock);__prep_account_new_huge_page (h, nid);spin_unlock_irq (&hugetlb_lock); } -static bool __prep_compound_gigantic_page (struct page *page,unsigned int order, -bool demote) +static bool __prep_compound_gigantic_folio (struct folio *folio, +unsigned int order,bool demote) {int i, j;int nr_pages =1 << order;struct page *p; -/* we rely on prep_new_huge_page to set the destructor */ -set_compound_order (page, order); -__SetPageHead (page); +/* we rely on prep_new_hugetlb_folio to set the destructor */ +folio_set_compound_order (folio, order); +__SetPageHead (&folio->page);for (i =0 ; i < nr_pages; i++) { - p =nth_page (page, i); + p =folio_page (folio, i);/* * For gigantic hugepages allocated through bootmem at
@@ -1851,43 +1849,41 @@ static bool __prep_compound_gigantic_page(struct page *page, unsigned int order,VM_BUG_ON_PAGE (page_count (p), p); }if (i !=0 ) -set_compound_head (p, page); +set_compound_head (p, &folio->page); } -atomic_set (compound_mapcount_ptr (page),-1 ); -atomic_set (subpages_mapcount_ptr (page),0 ); -atomic_set (compound_pincount_ptr (page),0 ); +atomic_set (folio_mapcount_ptr (folio),-1 ); +atomic_set (folio_subpages_mapcount_ptr (folio),0 ); +atomic_set (folio_pincount_ptr (folio),0 );return true; out_error:/* undo page modifications made above */ for (j =0 ; j < i; j++) { - p =nth_page (page, j); + p =folio_page (folio, j);if (j !=0 )clear_compound_head (p);set_page_refcounted (p); }/* need to clear PG_reserved on remaining tail pages */ for (; j < nr_pages; j++) { - p =nth_page (page, j); + p =folio_page (folio, j);__ClearPageReserved (p); } -set_compound_order (page,0 ); -#ifdef CONFIG_64BIT - page[1 ].compound_nr =0 ; -#endif -__ClearPageHead (page); +folio_set_compound_order (folio,0 ); +folio_clear_head (folio);return false; } -static bool prep_compound_gigantic_page (struct page *page,unsigned int order) +static bool prep_compound_gigantic_folio (struct folio *folio, +unsigned int order) { -return __prep_compound_gigantic_page (page, order, false); +return __prep_compound_gigantic_folio (folio, order, false); } -static bool prep_compound_gigantic_page_for_demote (struct page *page, +static bool prep_compound_gigantic_folio_for_demote (struct folio *folio,unsigned int order) { -return __prep_compound_gigantic_page (page, order, true); +return __prep_compound_gigantic_folio (folio, order, true); }/*
@@ -2039,7 +2035,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,return NULL ; folio =page_folio (page);if (hstate_is_gigantic (h)) { -if (!prep_compound_gigantic_page (page,huge_page_order (h))) { +if (!prep_compound_gigantic_folio (folio,huge_page_order (h))) {/* * Rare failure to convert pages to compound page. * Free pages and try again - ONCE!
@@ -2052,7 +2048,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,return NULL ; } } -prep_new_huge_page (h, page,page_to_nid (page)); +prep_new_hugetlb_folio (h, folio,folio_nid (folio));return page; }
@@ -3056,10 +3052,10 @@ static void __init gather_bootmem_prealloc(void)struct hstate *h = m->hstate;VM_BUG_ON (!hstate_is_gigantic (h)); -WARN_ON (page_count (page) !=1 ); -if (prep_compound_gigantic_page (page,huge_page_order (h))) { -WARN_ON (PageReserved (page)); -prep_new_huge_page (h, page,page_to_nid (page)); +WARN_ON (folio_ref_count (folio) !=1 ); +if (prep_compound_gigantic_folio (folio,huge_page_order (h))) { +WARN_ON (folio_test_reserved (folio)); +prep_new_hugetlb_folio (h, folio,folio_nid (folio));free_huge_page (page);/* add to the hugepage allocator */ }else {/* VERY unlikely inflated ref count on a tail page */
@@ -3478,13 +3474,14 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)for (i =0 ; i <pages_per_huge_page (h); i +=pages_per_huge_page (target_hstate)) { subpage =nth_page (page, i); + folio =page_folio (subpage);if (hstate_is_gigantic (target_hstate)) -prep_compound_gigantic_page_for_demote (subpage, +prep_compound_gigantic_folio_for_demote (folio, target_hstate->order);else prep_compound_page (subpage, target_hstate->order);set_page_private (subpage,0 ); -prep_new_huge_page (target_hstate, subpage, nid); +prep_new_hugetlb_folio (target_hstate, folio, nid);free_huge_page (subpage); }mutex_unlock (&target_hstate->resize_lock);
Signature · 1 line
2.38.1
PATCH v3 10/10 mm/hugetlb: change hugetlb allocation functions to return a folio
In reply to Sidhartha KumarMany hugetlb allocation helper functions have now been converting to folios, update their higher level callers to be compatible with folios.
--- mm/hugetlb.c | 98 ++++++++++++++++++++++++---------------------------- 1 file changed, 46 insertions(+), 52 deletions(-)
mm/hugetlb.c +46 -52
@@ -1378,7 +1378,7 @@ static void free_gigantic_folio(struct folio *folio, unsigned int order) }#ifdef CONFIG_CONTIG_ALLOC -static struct page *alloc_gigantic_page (struct hstate *h,gfp_t gfp_mask, +static struct folio *alloc_gigantic_folio (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nodemask) {unsigned long nr_pages =pages_per_huge_page (h);
@@ -1394,7 +1394,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, page =cma_alloc (hugetlb_cma[nid], nr_pages,huge_page_order (h), true);if (page) -return page; +return page_folio (page); }if (!(gfp_mask & __GFP_THISNODE)) {
@@ -1405,17 +1405,16 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask, page =cma_alloc (hugetlb_cma[node], nr_pages,huge_page_order (h), true);if (page) - return page; + return page_folio(page); } } }#endif - -return alloc_contig_pages (nr_pages, gfp_mask, nid, nodemask); +return page_folio (alloc_contig_pages (nr_pages, gfp_mask, nid, nodemask)); }#else /* !CONFIG_CONTIG_ALLOC */ -static struct page *alloc_gigantic_page (struct hstate *h,gfp_t gfp_mask, +static struct folio *alloc_gigantic_folio (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nodemask) {return NULL ;
@@ -1423,7 +1422,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,#endif /* CONFIG_CONTIG_ALLOC */ #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */ -static struct page *alloc_gigantic_page (struct hstate *h,gfp_t gfp_mask, +static struct folio *alloc_gigantic_folio (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nodemask) {return NULL ;
@@ -1948,7 +1947,7 @@ pgoff_t hugetlb_basepage_index(struct page *page)return (index <<compound_order (page_head)) + compound_idx; } -static struct page *alloc_buddy_huge_page (struct hstate *h, +static struct folio *alloc_buddy_hugetlb_folio (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nmask,nodemask_t *node_alloc_noretry) {
@@ -2007,7 +2006,7 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,if (node_alloc_noretry && !page && alloc_try_hard)node_set (nid, *node_alloc_noretry); -return page; +return page_folio (page); }/*
@@ -2017,23 +2016,21 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,* Note that returned page is 'frozen': ref count of head page and all tail * pages is zero. */ -static struct page *alloc_fresh_huge_page (struct hstate *h, +static struct folio *alloc_fresh_hugetlb_folio (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nmask,nodemask_t *node_alloc_noretry) { -struct page *page;struct folio *folio;bool retry = false; retry:if (hstate_is_gigantic (h)) - page =alloc_gigantic_page (h, gfp_mask, nid, nmask); + folio =alloc_gigantic_folio (h, gfp_mask, nid, nmask);else - page =alloc_buddy_huge_page (h, gfp_mask, + folio =alloc_buddy_hugetlb_folio (h, gfp_mask, nid, nmask, node_alloc_noretry); -if (!page) +if (!folio)return NULL ; - folio =page_folio (page);if (hstate_is_gigantic (h)) {if (!prep_compound_gigantic_folio (folio,huge_page_order (h))) {/*
@@ -2050,7 +2047,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h, }prep_new_hugetlb_folio (h, folio,folio_nid (folio)); -return page; +return folio; }/*
@@ -2060,21 +2057,21 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,static int alloc_pool_huge_page (struct hstate *h,nodemask_t *nodes_allowed,nodemask_t *node_alloc_noretry) { -struct page *page; +struct folio *folio;int nr_nodes, node;gfp_t gfp_mask =htlb_alloc_mask (h) | __GFP_THISNODE;for_each_node_mask_to_alloc (h, nr_nodes, node, nodes_allowed) { - page =alloc_fresh_huge_page (h, gfp_mask, node, nodes_allowed, - node_alloc_noretry); -if (page) + folio =alloc_fresh_hugetlb_folio (h, gfp_mask, node, + nodes_allowed, node_alloc_noretry); +if (folio)break ; } -if (!page) +if (!folio)return 0 ; -free_huge_page (page);/* free it into the hugepage allocator */ +free_huge_page (&folio->page);/* free it into the hugepage allocator */ return 1 ; }
@@ -2235,7 +2232,7 @@ int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)static struct page *alloc_surplus_huge_page (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nmask) { -struct page *page =NULL ; +struct folio *folio =NULL ;if (hstate_is_gigantic (h))return NULL ;
@@ -2245,8 +2242,8 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, goto out_unlock;spin_unlock_irq (&hugetlb_lock); - page =alloc_fresh_huge_page (h, gfp_mask, nid, nmask,NULL ); -if (!page) + folio =alloc_fresh_hugetlb_folio (h, gfp_mask, nid, nmask,NULL ); +if (!folio)return NULL ;spin_lock_irq (&hugetlb_lock);
@@ -2258,43 +2255,42 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,* codeflow */ if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { -SetHPageTemporary (page); +folio_set_hugetlb_temporary (folio);spin_unlock_irq (&hugetlb_lock); -free_huge_page (page); +free_huge_page (&folio->page);return NULL ; } h->surplus_huge_pages++; - h->surplus_huge_pages_node[page_to_nid (page)]++; + h->surplus_huge_pages_node[folio_nid (folio)]++; out_unlock:spin_unlock_irq (&hugetlb_lock); -return page; +return &folio->page; }static struct page *alloc_migrate_huge_page (struct hstate *h,gfp_t gfp_mask,int nid,nodemask_t *nmask) { -struct page *page; +struct folio *folio;if (hstate_is_gigantic (h))return NULL ; - page =alloc_fresh_huge_page (h, gfp_mask, nid, nmask,NULL ); -if (!page) + folio =alloc_fresh_hugetlb_folio (h, gfp_mask, nid, nmask,NULL ); +if (!folio)return NULL ;/* fresh huge pages are frozen */ -set_page_refcounted (page); - +folio_ref_unfreeze (folio,1 );/* * We do not account these pages as surplus because they are only * temporary and will be released properly on the last reference */ -SetHPageTemporary (page); +folio_set_hugetlb_temporary (folio); -return page; +return &folio->page; }/*
@@ -2743,19 +2739,18 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, }/* -* alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one +* alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve +* the old one * @h: struct hstate old page belongs to * @old_page: Old page to dissolve * @list: List to isolate the page in case we need to * Returns 0 on success, otherwise negated error. */ -static int alloc_and_dissolve_huge_page (struct hstate *h,struct page *old_page, -struct list_head *list) +static int alloc_and_dissolve_hugetlb_folio (struct hstate *h, +struct folio *old_folio,struct list_head *list) {gfp_t gfp_mask =htlb_alloc_mask (h) | __GFP_THISNODE; -struct folio *old_folio =page_folio (old_page);int nid =folio_nid (old_folio); -struct page *new_page;struct folio *new_folio;int ret =0 ;
@@ -2766,26 +2761,25 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,* the pool. This simplifies and let us do most of the processing * under the lock. */ - new_page =alloc_buddy_huge_page (h, gfp_mask, nid,NULL ,NULL ); -if (!new_page) + new_folio =alloc_buddy_hugetlb_folio (h, gfp_mask, nid,NULL ,NULL ); +if (!new_folio)return -ENOMEM ; - new_folio =page_folio (new_page);__prep_new_hugetlb_folio (h, new_folio); retry:spin_lock_irq (&hugetlb_lock);if (!folio_test_hugetlb (old_folio)) {/* -* Freed from under us. Drop new_page too. +* Freed from under us. Drop new_folio too. */ goto free_new; }else if (folio_ref_count (old_folio)) {/* -* Someone has grabbed the page, try to isolate it here. +* Someone has grabbed the folio, try to isolate it here. * Fail with -EBUSY if not possible. */ spin_unlock_irq (&hugetlb_lock); - ret =isolate_hugetlb (old_page, list); + ret =isolate_hugetlb (&old_folio->page, list);spin_lock_irq (&hugetlb_lock); goto free_new; }else if (!folio_test_hugetlb_freed (old_folio)) {
@@ -2863,7 +2857,7 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)if (folio_ref_count (folio) && !isolate_hugetlb (&folio->page, list)) ret =0 ;else if (!folio_ref_count (folio)) - ret =alloc_and_dissolve_huge_page (h, &folio->page, list); + ret =alloc_and_dissolve_hugetlb_folio (h, folio, list);return ret; }
@@ -3081,14 +3075,14 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)if (!alloc_bootmem_huge_page (h, nid))break ; }else { -struct page *page; +struct folio *folio;gfp_t gfp_mask =htlb_alloc_mask (h) | __GFP_THISNODE; - page =alloc_fresh_huge_page (h, gfp_mask, nid, + folio =alloc_fresh_hugetlb_folio (h, gfp_mask, nid, &node_states[N_MEMORY ],NULL ); -if (!page) +if (!folio)break ; -free_huge_page (page);/* free it into the hugepage allocator */ +free_huge_page (&folio->page);/* free it into the hugepage allocator */ }cond_resched (); }
Signature · 1 line
2.38.1