mm/hugetlb: alloc/free gigantic folios
Subsystems (5)
cover mm/hugetlb: alloc/free gigantic folios
Use __GFP_COMP for gigantic folios can greatly reduce not only the complexity in the code but also the allocation and free time.
Approximate LOC to mm/hugetlb.c: -200, +50
Allocate and free 500 1GB hugeTLB memory without HVO by:
time echo 500 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages time echo 0 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
Before After Alloc ~13s ~10s Free ~15s <1s
The above magnitude generally holds for multiple x86 and arm64 CPU models.
Yu Zhao (3):
mm/contig_alloc: support __GFP_COMP
mm/cma: add cma_alloc_folio()
mm/hugetlb: use __GFP_COMP for gigantic folios
include/linux/cma.h | 1 + include/linux/hugetlb.h | 9 +- mm/cma.c | 47 +++++--- mm/compaction.c | 48 +------- mm/hugetlb.c | 244 ++++++++-------------------------------- mm/internal.h | 9 ++ mm/page_alloc.c | 111 +++++++++++++----- 7 files changed, 177 insertions(+), 292 deletions(-)
base-commit: b447504e1fed49fabbc03d6c2530126824f87c92
prerequisite-patch-id: 9fe502f7c87a9f951d0aee61f426bd85bc43ef74
Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
PATCH 1/3 mm/contig_alloc: support __GFP_COMP
Support __GFP_COMP in alloc_contig_range(). When the flag is set, upon success the function returns a large folio prepared by prep_new_page(), rather than a range of order-0 pages prepared by split_free_pages() (which is renamed from split_map_pages()).
alloc_contig_range() can return folios larger than MAX_PAGE_ORDER, e.g., gigantic hugeTLB folios. As a result, on the free path free_one_page() needs to handle this case by split_large_buddy(), in addition to free_contig_range() properly handling large folios by folio_put().
--- mm/compaction.c | 48 +++------------------ mm/internal.h | 9 ++++ mm/page_alloc.c | 111 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 94 insertions(+), 74 deletions(-)
mm/compaction.c +5 -43
@@ -79,40 +79,6 @@ static inline bool is_via_compact_memory(int order) { return false; }#define COMPACTION_HPAGE_ORDER (PMD_SHIFT - PAGE_SHIFT)#endif -static struct page *mark_allocated_noprof (struct page *page,unsigned int order,gfp_t gfp_flags) -{ -post_alloc_hook (page, order, __GFP_MOVABLE); -return page; -} -#define mark_allocated (...) alloc_hooks(mark_allocated_noprof(__VA_ARGS__)) - -static void split_map_pages (struct list_head *freepages) -{ -unsigned int i, order; -struct page *page, *next; -LIST_HEAD (tmp_list); - -for (order =0 ; order <NR_PAGE_ORDERS ; order++) { -list_for_each_entry_safe (page, next, &freepages[order], lru) { -unsigned int nr_pages; - -list_del (&page->lru); - - nr_pages =1 << order; - -mark_allocated (page, order, __GFP_MOVABLE); -if (order) -split_page (page, order); - -for (i =0 ; i < nr_pages; i++) { -list_add (&page->lru, &tmp_list); - page++; - } - } -list_splice_init (&tmp_list, &freepages[0 ]); - } -} -static unsigned long release_free_list (struct list_head *freepages) {int order;
@@ -742,11 +708,11 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,* * Non-free pages, invalid PFNs, or zone boundaries within the * [start_pfn, end_pfn) range are considered errors, cause function to -* undo its actions and return zero. +* undo its actions and return zero. cc->freepages[] are empty. * * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater then end_pfn if end fell in a middle of -* a free page). +* a free page). cc->freepages[] contain free pages isolated. */ unsigned long isolate_freepages_range(struct compact_control *cc,
@@ -754,10 +720,9 @@ isolate_freepages_range(struct compact_control *cc, {unsigned long isolated, pfn, block_start_pfn, block_end_pfn;int order; -struct list_head tmp_freepages[NR_PAGE_ORDERS ];for (order =0 ; order <NR_PAGE_ORDERS ; order++) -INIT_LIST_HEAD (&tmp_freepages[order]); +INIT_LIST_HEAD (&cc->freepages[order]); pfn = start_pfn; block_start_pfn =pageblock_start_pfn (pfn);
@@ -788,7 +753,7 @@ isolate_freepages_range(struct compact_control *cc,break ; isolated =isolate_freepages_block (cc, &isolate_start_pfn, - block_end_pfn, tmp_freepages,0 , true); + block_end_pfn, cc->freepages,0 , true);/* * In strict mode, isolate_freepages_block() returns 0 if
@@ -807,13 +772,10 @@ isolate_freepages_range(struct compact_control *cc,if (pfn < end_pfn) {/* Loop terminated early, cleanup. */ -release_free_list (tmp_freepages); +release_free_list (cc->freepages);return 0 ; } -/* __isolate_free_page() does not map the pages */ -split_map_pages (tmp_freepages); -/* We don't use freelists for anything. */ return pfn; }
mm/internal.h +9 -0
@@ -679,6 +679,15 @@ extern void prep_compound_page(struct page *page, unsigned int order);extern void post_alloc_hook (struct page *page,unsigned int order,gfp_t gfp_flags); + +static inline struct page *post_alloc_hook_noprof (struct page *page,unsigned int order, +gfp_t gfp_flags) +{ +post_alloc_hook (page, order, __GFP_MOVABLE); +return page; +} +#define mark_allocated (...) alloc_hooks(post_alloc_hook_noprof(__VA_ARGS__)) +extern bool free_pages_prepare (struct page *page,unsigned int order);extern int user_min_free_kbytes;
mm/page_alloc.c +80 -31
161 lines not on this page · raw message
Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
PATCH 2/3 mm/cma: add cma_alloc_folio()
In reply to Yu ZhaoWith alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().
--- include/linux/cma.h | 1 + mm/cma.c | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma);extern struct page *cma_alloc (struct cma *cma,unsigned long count,unsigned int align,bool no_warn); +extern struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp);extern bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count);extern bool cma_release (struct cma *cma,const struct page *pages,unsigned long count);
mm/cma.c +32 -15
71 lines not on this page · raw message
Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()
Quoted 86 lines · Yu Zhao
With
alloc_contig_range()andfree_contig_range()supporting large folios, CMA can allocate and free large folios too, bycma_alloc_folio()andcma_release().Signed-off-by Yu Zhao <[email protected]>--- include/linux/cma.h | 1 + mm/cma.c | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-)include/linux/cma.h +1
-0@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma);extern struct page *cma_alloc (struct cma *cma,unsigned long count,unsigned int align,bool no_warn); +extern struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp);extern bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count);extern bool cma_release (struct cma *cma,const struct page *pages,unsigned long count);mm/cma.c +20
-15@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)spin_unlock_irq (&cma->lock); } -/** -* cma_alloc() - allocate pages from contiguous area -* @cma: Contiguous memory region for which the allocation is performed. -* @count: Requested number of pages. -* @align: Requested alignment of pages (in PAGE_SIZE order). -* @no_warn: Avoid printing message about failed allocation -* -* This function allocates part of contiguous memory on specific -* contiguous memory area. -*/ -struct page *cma_alloc (struct cma *cma,unsigned long count, -unsigned int align,bool no_warn) +static struct page *__cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,gfp_t gfp) {unsigned long mask, offset;unsigned long pfn =-1 ;@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);mutex_lock (&cma_mutex); - ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , -GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); + ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , gfp);mutex_unlock (&cma_mutex);if (ret ==0 ) { page =pfn_to_page (pfn);@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,page_kasan_tag_reset (nth_page (page, i)); } -if (ret && !no_warn) { +if (ret && !(gfp & __GFP_NOWARN)) {pr_err_ratelimited ("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n" , __func__, cma->name, count, ret);cma_debug_show_areas (cma);@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,return page; } +/** +* cma_alloc() - allocate pages from contiguous area +* @cma: Contiguous memory region for which the allocation is performed. +* @count: Requested number of pages. +* @align: Requested alignment of pages (in PAGE_SIZE order). +* @no_warn: Avoid printing message about failed allocation +* +* This function allocates part of contiguous memory on specific +* contiguous memory area. +*/ +struct page *cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,bool no_warn) +{ +return __cma_alloc (cma, count, align,GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); +} +
mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530
Quoted 10 lines
mm/cma.c +10
-0+struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp) +{ +struct page *page; + +if (WARN_ON (order && !(gfp | __GFP_COMP))) +return NULL ; + + page =__cma_alloc (cma,1 << order, order, gfp); + +return page ?page_folio (page) :NULL ;
We don't set large_rmappable for cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must without large_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we set large_rmappable here, and clear it in init_new_hugetlb_folio()?
Quoted 5 lines
mm/cma.c +2
-0+} +bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count) {
Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()
Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao
With
alloc_contig_range()andfree_contig_range()supporting large folios, CMA can allocate and free large folios too, bycma_alloc_folio()andcma_release().Signed-off-by Yu Zhao <[email protected]>--- include/linux/cma.h | 1 + mm/cma.c | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-)include/linux/cma.h +1
-0@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma);extern struct page *cma_alloc (struct cma *cma,unsigned long count,unsigned int align,bool no_warn); +extern struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp);extern bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count);extern bool cma_release (struct cma *cma,const struct page *pages,unsigned long count);mm/cma.c +29
-15@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)spin_unlock_irq (&cma->lock); } -/** -* cma_alloc() - allocate pages from contiguous area -* @cma: Contiguous memory region for which the allocation is performed. -* @count: Requested number of pages. -* @align: Requested alignment of pages (in PAGE_SIZE order). -* @no_warn: Avoid printing message about failed allocation -* -* This function allocates part of contiguous memory on specific -* contiguous memory area. -*/ -struct page *cma_alloc (struct cma *cma,unsigned long count, -unsigned int align,bool no_warn) +static struct page *__cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,gfp_t gfp) {unsigned long mask, offset;unsigned long pfn =-1 ;@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);mutex_lock (&cma_mutex); - ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , -GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); + ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , gfp);mutex_unlock (&cma_mutex);if (ret ==0 ) { page =pfn_to_page (pfn);@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,page_kasan_tag_reset (nth_page (page, i)); } -if (ret && !no_warn) { +if (ret && !(gfp & __GFP_NOWARN)) {pr_err_ratelimited ("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n" , __func__, cma->name, count, ret);cma_debug_show_areas (cma);@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,return page; } +/** +* cma_alloc() - allocate pages from contiguous area +* @cma: Contiguous memory region for which the allocation is performed. +* @count: Requested number of pages. +* @align: Requested alignment of pages (in PAGE_SIZE order). +* @no_warn: Avoid printing message about failed allocation +* +* This function allocates part of contiguous memory on specific +* contiguous memory area. +*/ +struct page *cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,bool no_warn) +{ +return __cma_alloc (cma, count, align,GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); +} + +struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp) +{ +struct page *page; + +if (WARN_ON (order && !(gfp | __GFP_COMP))) +return NULL ; + + page =__cma_alloc (cma,1 << order, order, gfp); +
Quoted 8 lines
mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530
Quoted 1 line
mm/cma.c +1
-0+return page ?page_folio (page) :NULL ;We don't set
large_rmappablefor cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must withoutlarge_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we setlarge_rmappablehere, and clear it ininit_new_hugetlb_folio()?
I want to hear what Matthew thinks about this.
My opinion is that we don't want to couple largely rmappable (or deferred splittable) with __GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.
Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.
Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()
Quoted 117 lines · Yu Zhao
Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao
With
alloc_contig_range()andfree_contig_range()supporting large folios, CMA can allocate and free large folios too, bycma_alloc_folio()andcma_release().Signed-off-by Yu Zhao <[email protected]>--- include/linux/cma.h | 1 + mm/cma.c | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-)include/linux/cma.h +1
-0@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma);extern struct page *cma_alloc (struct cma *cma,unsigned long count,unsigned int align,bool no_warn); +extern struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp);extern bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count);extern bool cma_release (struct cma *cma,const struct page *pages,unsigned long count);mm/cma.c +29
-15@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)spin_unlock_irq (&cma->lock); } -/** -* cma_alloc() - allocate pages from contiguous area -* @cma: Contiguous memory region for which the allocation is performed. -* @count: Requested number of pages. -* @align: Requested alignment of pages (in PAGE_SIZE order). -* @no_warn: Avoid printing message about failed allocation -* -* This function allocates part of contiguous memory on specific -* contiguous memory area. -*/ -struct page *cma_alloc (struct cma *cma,unsigned long count, -unsigned int align,bool no_warn) +static struct page *__cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,gfp_t gfp) {unsigned long mask, offset;unsigned long pfn =-1 ;@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);mutex_lock (&cma_mutex); - ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , -GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); + ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , gfp);mutex_unlock (&cma_mutex);if (ret ==0 ) { page =pfn_to_page (pfn);@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,page_kasan_tag_reset (nth_page (page, i)); } -if (ret && !no_warn) { +if (ret && !(gfp & __GFP_NOWARN)) {pr_err_ratelimited ("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n" , __func__, cma->name, count, ret);cma_debug_show_areas (cma);@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,return page; } +/** +* cma_alloc() - allocate pages from contiguous area +* @cma: Contiguous memory region for which the allocation is performed. +* @count: Requested number of pages. +* @align: Requested alignment of pages (in PAGE_SIZE order). +* @no_warn: Avoid printing message about failed allocation +* +* This function allocates part of contiguous memory on specific +* contiguous memory area. +*/ +struct page *cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,bool no_warn) +{ +return __cma_alloc (cma, count, align,GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); +} + +struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp) +{ +struct page *page; + +if (WARN_ON (order && !(gfp | __GFP_COMP))) +return NULL ; + + page =__cma_alloc (cma,1 << order, order, gfp); +Quoted 8 lines
mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530
Quoted 1 line
mm/cma.c +1
-0+return page ?page_folio (page) :NULL ;We don't set
large_rmappablefor cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must withoutlarge_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we setlarge_rmappablehere, and clear it ininit_new_hugetlb_folio()?I want to hear what Matthew thinks about this.
My opinion is that we don't want to couple largely rmappable (or deferred splittable) with
__GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.
Quoted 6 lines
Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.
Also along the similar lines, Usama is trying to add PG_partially_mapped [1], which I have explicitly asked him not to introduce that flag to hugeTLB, unless there are good reasons (none ATM).
[1] https://lore.kernel.org/CAOUHufbmgwZwzUuHVvEDMqPGcsxE2hEreRZ4PhK5yz27GdK-Tw@mail.gmail.com/
Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()
Quoted 123 lines · Yu Zhao
Quoted 122 lines · Yu Zhao
Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao
With
alloc_contig_range()andfree_contig_range()supporting large folios, CMA can allocate and free large folios too, bycma_alloc_folio()andcma_release().Signed-off-by Yu Zhao <[email protected]>--- include/linux/cma.h | 1 + mm/cma.c | 47 ++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-)include/linux/cma.h +1
-0@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma);extern struct page *cma_alloc (struct cma *cma,unsigned long count,unsigned int align,bool no_warn); +extern struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp);extern bool cma_pages_valid (struct cma *cma,const struct page *pages,unsigned long count);extern bool cma_release (struct cma *cma,const struct page *pages,unsigned long count);mm/cma.c +29
-15@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)spin_unlock_irq (&cma->lock); } -/** -* cma_alloc() - allocate pages from contiguous area -* @cma: Contiguous memory region for which the allocation is performed. -* @count: Requested number of pages. -* @align: Requested alignment of pages (in PAGE_SIZE order). -* @no_warn: Avoid printing message about failed allocation -* -* This function allocates part of contiguous memory on specific -* contiguous memory area. -*/ -struct page *cma_alloc (struct cma *cma,unsigned long count, -unsigned int align,bool no_warn) +static struct page *__cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,gfp_t gfp) {unsigned long mask, offset;unsigned long pfn =-1 ;@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);mutex_lock (&cma_mutex); - ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , -GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); + ret =alloc_contig_range (pfn, pfn + count,MIGRATE_CMA , gfp);mutex_unlock (&cma_mutex);if (ret ==0 ) { page =pfn_to_page (pfn);@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,page_kasan_tag_reset (nth_page (page, i)); } -if (ret && !no_warn) { +if (ret && !(gfp & __GFP_NOWARN)) {pr_err_ratelimited ("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n" , __func__, cma->name, count, ret);cma_debug_show_areas (cma);@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,return page; } +/** +* cma_alloc() - allocate pages from contiguous area +* @cma: Contiguous memory region for which the allocation is performed. +* @count: Requested number of pages. +* @align: Requested alignment of pages (in PAGE_SIZE order). +* @no_warn: Avoid printing message about failed allocation +* +* This function allocates part of contiguous memory on specific +* contiguous memory area. +*/ +struct page *cma_alloc (struct cma *cma,unsigned long count, +unsigned int align,bool no_warn) +{ +return __cma_alloc (cma, count, align,GFP_KERNEL | (no_warn ? __GFP_NOWARN :0 )); +} + +struct folio *cma_alloc_folio (struct cma *cma,int order,gfp_t gfp) +{ +struct page *page; + +if (WARN_ON (order && !(gfp | __GFP_COMP))) +return NULL ; + + page =__cma_alloc (cma,1 << order, order, gfp); +Quoted 8 lines
mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530
Quoted 1 line
mm/cma.c +1
-0+return page ?page_folio (page) :NULL ;We don't set
large_rmappablefor cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must withoutlarge_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we setlarge_rmappablehere, and clear it ininit_new_hugetlb_folio()?I want to hear what Matthew thinks about this.
My opinion is that we don't want to couple largely rmappable (or deferred splittable) with
__GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.
Quoted 8 lines
Quoted 1 line
Also along the similar lines, Usama is trying to add
PG_partially_mapped[1], which I have explicitly asked him not to introduce that flag to hugeTLB, unless there are good reasons (none ATM).[1] https://lore.kernel.org/CAOUHufbmgwZwzUuHVvEDMqPGcsxE2hEreRZ4PhK5yz27GdK-Tw@mail.gmail.com/
PG_partially_mapped won't be cleared for hugeTLB in the next revision of the series as suggested by Yu. Its not there in the fix patch I posted as well in https://lore.kernel.org/all/[email protected]/
PATCH 3/3 mm/hugetlb: use __GFP_COMP for gigantic folios
In reply to Yu ZhaoUse __GFP_COMP for gigantic folios to greatly reduce not only the code but also the allocation and free time.
LOC (approximately): -200, +50
Allocate and free 500 1GB hugeTLB memory without HVO by:
time echo 500 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages time echo 0 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
Before After Alloc ~13s ~10s Free ~15s <1s
The above magnitude generally holds for multiple x86 and arm64 CPU models.
--- include/linux/hugetlb.h | 9 +- mm/hugetlb.c | 244 ++++++++-------------------------------- 2 files changed, 50 insertions(+), 203 deletions(-)
include/linux/hugetlb.h +5 -4
@@ -896,10 +896,11 @@ static inline bool hugepage_movable_supported(struct hstate *h)/* Movability of hugepages depends on migration support. */ static inline gfp_t htlb_alloc_mask (struct hstate *h) { -if (hugepage_movable_supported (h)) -return GFP_HIGHUSER_MOVABLE ; -else -return GFP_HIGHUSER ; +gfp_t gfp = __GFP_COMP | __GFP_NOWARN; + + gfp |=hugepage_movable_supported (h) ?GFP_HIGHUSER_MOVABLE :GFP_HIGHUSER ; + +return gfp; }static inline gfp_t htlb_modify_alloc_mask (struct hstate *h,gfp_t gfp_mask)
mm/hugetlb.c +45 -199
346 lines not on this page · raw message
Signature · 1 line
2.46.0.76.ge559c4bf1a-goog