Skip to contentSkip to conversation

mm/hugetlb: alloc/free gigantic folios

PATCH Superseded Yu Zhao 3 participants 8 messages Aug 11, 2024 – Aug 15, 2024

Subsystems (5)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Yu Zhao

author
#0 lore raw

cover mm/hugetlb: alloc/free gigantic folios

Use __GFP_COMP for gigantic folios can greatly reduce not only the complexity in the code but also the allocation and free time.

Approximate LOC to mm/hugetlb.c: -200, +50

Allocate and free 500 1GB hugeTLB memory without HVO by:
time echo 500 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages time echo 0 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages

       Before  After
Alloc  ~13s    ~10s
Free   ~15s    <1s

The above magnitude generally holds for multiple x86 and arm64 CPU models.

Yu Zhao (3):
mm/contig_alloc: support __GFP_COMP
mm/cma: add cma_alloc_folio()
mm/hugetlb: use __GFP_COMP for gigantic folios

 include/linux/cma.h     |   1 +
 include/linux/hugetlb.h |   9 +-
 mm/cma.c                |  47 +++++---
 mm/compaction.c         |  48 +-------
 mm/hugetlb.c            | 244 ++++++++--------------------------------
 mm/internal.h           |   9 ++
 mm/page_alloc.c         | 111 +++++++++++++-----
 7 files changed, 177 insertions(+), 292 deletions(-)

base-commit: b447504e1fed49fabbc03d6c2530126824f87c92
prerequisite-patch-id: 9fe502f7c87a9f951d0aee61f426bd85bc43ef74

Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
author
#1 lore raw

PATCH 1/3 mm/contig_alloc: support __GFP_COMP

Support __GFP_COMP in alloc_contig_range(). When the flag is set, upon success the function returns a large folio prepared by prep_new_page(), rather than a range of order-0 pages prepared by split_free_pages() (which is renamed from split_map_pages()).

alloc_contig_range() can return folios larger than MAX_PAGE_ORDER, e.g., gigantic hugeTLB folios. As a result, on the free path free_one_page() needs to handle this case by split_large_buddy(), in addition to free_contig_range() properly handling large folios by folio_put().

---
 mm/compaction.c |  48 +++------------------
 mm/internal.h   |   9 ++++
 mm/page_alloc.c | 111 ++++++++++++++++++++++++++++++++++--------------
 3 files changed, 94 insertions(+), 74 deletions(-)
mm/compaction.c +5 -43
index eb95e9b435d0..1ebfef98e1d0 100644
@@ -79,40 +79,6 @@ static inline bool is_via_compact_memory(int order) { return false; }
 #define COMPACTION_HPAGE_ORDER	(PMD_SHIFT - PAGE_SHIFT)
 #endif
 
-static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)
-{
-	post_alloc_hook(page, order, __GFP_MOVABLE);
-	return page;
-}
-#define mark_allocated(...)	alloc_hooks(mark_allocated_noprof(__VA_ARGS__))
-
-static void split_map_pages(struct list_head *freepages)
-{
-	unsigned int i, order;
-	struct page *page, *next;
-	LIST_HEAD(tmp_list);
-
-	for (order = 0; order < NR_PAGE_ORDERS; order++) {
-		list_for_each_entry_safe(page, next, &freepages[order], lru) {
-			unsigned int nr_pages;
-
-			list_del(&page->lru);
-
-			nr_pages = 1 << order;
-
-			mark_allocated(page, order, __GFP_MOVABLE);
-			if (order)
-				split_page(page, order);
-
-			for (i = 0; i < nr_pages; i++) {
-				list_add(&page->lru, &tmp_list);
-				page++;
-			}
-		}
-		list_splice_init(&tmp_list, &freepages[0]);
-	}
-}
-
 static unsigned long release_free_list(struct list_head *freepages)
 {
 	int order;
@@ -742,11 +708,11 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
  *
  * Non-free pages, invalid PFNs, or zone boundaries within the
  * [start_pfn, end_pfn) range are considered errors, cause function to
- * undo its actions and return zero.
+ * undo its actions and return zero. cc->freepages[] are empty.
  *
  * Otherwise, function returns one-past-the-last PFN of isolated page
  * (which may be greater then end_pfn if end fell in a middle of
- * a free page).
+ * a free page). cc->freepages[] contain free pages isolated.
  */
 unsigned long
 isolate_freepages_range(struct compact_control *cc,
@@ -754,10 +720,9 @@ isolate_freepages_range(struct compact_control *cc,
 {
 	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
 	int order;
-	struct list_head tmp_freepages[NR_PAGE_ORDERS];
 
 	for (order = 0; order < NR_PAGE_ORDERS; order++)
-		INIT_LIST_HEAD(&tmp_freepages[order]);
+		INIT_LIST_HEAD(&cc->freepages[order]);
 
 	pfn = start_pfn;
 	block_start_pfn = pageblock_start_pfn(pfn);
@@ -788,7 +753,7 @@ isolate_freepages_range(struct compact_control *cc,
 			break;
 
 		isolated = isolate_freepages_block(cc, &isolate_start_pfn,
-					block_end_pfn, tmp_freepages, 0, true);
+					block_end_pfn, cc->freepages, 0, true);
 
 		/*
 		 * In strict mode, isolate_freepages_block() returns 0 if
@@ -807,13 +772,10 @@ isolate_freepages_range(struct compact_control *cc,
 
 	if (pfn < end_pfn) {
 		/* Loop terminated early, cleanup. */
-		release_free_list(tmp_freepages);
+		release_free_list(cc->freepages);
 		return 0;
 	}
 
-	/* __isolate_free_page() does not map the pages */
-	split_map_pages(tmp_freepages);
-
 	/* We don't use freelists for anything. */
 	return pfn;
 }
mm/internal.h +9 -0
index acda347620c6..03e795ce755f 100644
@@ -679,6 +679,15 @@ extern void prep_compound_page(struct page *page, unsigned int order);
 
 extern void post_alloc_hook(struct page *page, unsigned int order,
 					gfp_t gfp_flags);
+
+static inline struct page *post_alloc_hook_noprof(struct page *page, unsigned int order,
+						  gfp_t gfp_flags)
+{
+	post_alloc_hook(page, order, __GFP_MOVABLE);
+	return page;
+}
+#define mark_allocated(...) alloc_hooks(post_alloc_hook_noprof(__VA_ARGS__))
+
 extern bool free_pages_prepare(struct page *page, unsigned int order);
 
 extern int user_min_free_kbytes;
mm/page_alloc.c +80 -31

161 lines not on this page · raw message

Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
author
#2 lore raw

PATCH 2/3 mm/cma: add cma_alloc_folio()

In reply to Yu Zhao

With alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().

---
 include/linux/cma.h |  1 +
 mm/cma.c            | 47 ++++++++++++++++++++++++++++++---------------
 2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
index 9db877506ea8..086553fbda73 100644
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
 					struct cma **res_cma);
 extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
 			      bool no_warn);
+extern struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
 extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned long count);
 extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
mm/cma.c +32 -15

71 lines not on this page · raw message

Signature · 1 line
2.46.0.76.ge559c4bf1a-goog
#3 lore raw

Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()

Quoted 86 lines · Yu Zhao

With alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().

---
  include/linux/cma.h |  1 +
  mm/cma.c            | 47 ++++++++++++++++++++++++++++++---------------
  2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
index 9db877506ea8..086553fbda73 100644
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
  					struct cma **res_cma);
  extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
  			      bool no_warn);
+extern struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
  extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned long count);
  extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
mm/cma.c +20 -15
index 95d6950e177b..46feb06db8e7 100644
@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)
  	spin_unlock_irq(&cma->lock);
  }
 
-/**
- * cma_alloc() - allocate pages from contiguous area
- * @cma:   Contiguous memory region for which the allocation is performed.
- * @count: Requested number of pages.
- * @align: Requested alignment of pages (in PAGE_SIZE order).
- * @no_warn: Avoid printing message about failed allocation
- *
- * This function allocates part of contiguous memory on specific
- * contiguous memory area.
- */
-struct page *cma_alloc(struct cma *cma, unsigned long count,
-		       unsigned int align, bool no_warn)
+static struct page *__cma_alloc(struct cma *cma, unsigned long count,
+				unsigned int align, gfp_t gfp)
  {
  	unsigned long mask, offset;
  	unsigned long pfn = -1;
@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 
  		pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
  		mutex_lock(&cma_mutex);
-		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
-				     GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp);
  		mutex_unlock(&cma_mutex);
  		if (ret == 0) {
  			page = pfn_to_page(pfn);
@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
  			page_kasan_tag_reset(nth_page(page, i));
  	}
 
-	if (ret && !no_warn) {
+	if (ret && !(gfp & __GFP_NOWARN)) {
  		pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
  				   __func__, cma->name, count, ret);
  		cma_debug_show_areas(cma);
@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
  	return page;
  }
 
+/**
+ * cma_alloc() - allocate pages from contiguous area
+ * @cma:   Contiguous memory region for which the allocation is performed.
+ * @count: Requested number of pages.
+ * @align: Requested alignment of pages (in PAGE_SIZE order).
+ * @no_warn: Avoid printing message about failed allocation
+ *
+ * This function allocates part of contiguous memory on specific
+ * contiguous memory area.
+ */
+struct page *cma_alloc(struct cma *cma, unsigned long count,
+		       unsigned int align, bool no_warn)
+{
+	return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+}
+

mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530

Quoted 10 lines
mm/cma.c +10 -0
+struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
+{
+	struct page *page;
+
+	if (WARN_ON(order && !(gfp | __GFP_COMP)))
+		return NULL;
+
+	page = __cma_alloc(cma, 1 << order, order, gfp);
+
+	return page ? page_folio(page) : NULL;

We don't set large_rmappable for cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must without large_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we set large_rmappable here, and clear it in init_new_hugetlb_folio()?

Quoted 5 lines
mm/cma.c +2 -0
+}
+
  bool cma_pages_valid(struct cma *cma, const struct page *pages,
  		     unsigned long count)
  {
author
#4 lore raw

Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()

Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao

With alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().

---
  include/linux/cma.h |  1 +
  mm/cma.c            | 47 ++++++++++++++++++++++++++++++---------------
  2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
index 9db877506ea8..086553fbda73 100644
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
                                      struct cma **res_cma);
  extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
                            bool no_warn);
+extern struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
  extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned long count);
  extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
mm/cma.c +29 -15
index 95d6950e177b..46feb06db8e7 100644
@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)
      spin_unlock_irq(&cma->lock);
  }
 
-/**
- * cma_alloc() - allocate pages from contiguous area
- * @cma:   Contiguous memory region for which the allocation is performed.
- * @count: Requested number of pages.
- * @align: Requested alignment of pages (in PAGE_SIZE order).
- * @no_warn: Avoid printing message about failed allocation
- *
- * This function allocates part of contiguous memory on specific
- * contiguous memory area.
- */
-struct page *cma_alloc(struct cma *cma, unsigned long count,
-                    unsigned int align, bool no_warn)
+static struct page *__cma_alloc(struct cma *cma, unsigned long count,
+                             unsigned int align, gfp_t gfp)
  {
      unsigned long mask, offset;
      unsigned long pfn = -1;
@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 
              pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
              mutex_lock(&cma_mutex);
-             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
-                                  GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp);
              mutex_unlock(&cma_mutex);
              if (ret == 0) {
                      page = pfn_to_page(pfn);
@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
                      page_kasan_tag_reset(nth_page(page, i));
      }
 
-     if (ret && !no_warn) {
+     if (ret && !(gfp & __GFP_NOWARN)) {
              pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
                                 __func__, cma->name, count, ret);
              cma_debug_show_areas(cma);
@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
      return page;
  }
 
+/**
+ * cma_alloc() - allocate pages from contiguous area
+ * @cma:   Contiguous memory region for which the allocation is performed.
+ * @count: Requested number of pages.
+ * @align: Requested alignment of pages (in PAGE_SIZE order).
+ * @no_warn: Avoid printing message about failed allocation
+ *
+ * This function allocates part of contiguous memory on specific
+ * contiguous memory area.
+ */
+struct page *cma_alloc(struct cma *cma, unsigned long count,
+                    unsigned int align, bool no_warn)
+{
+     return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+}
+
+struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
+{
+     struct page *page;
+
+     if (WARN_ON(order && !(gfp | __GFP_COMP)))
+             return NULL;
+
+     page = __cma_alloc(cma, 1 << order, order, gfp);
+
Quoted 8 lines

mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530

Quoted 1 line
mm/cma.c +1 -0
+     return page ? page_folio(page) : NULL;

We don't set large_rmappable for cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must without large_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we set large_rmappable here, and clear it in init_new_hugetlb_folio()?

I want to hear what Matthew thinks about this.

My opinion is that we don't want to couple largely rmappable (or deferred splittable) with __GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.

Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.

[1] https://lpc.events/event/18/sessions/184/#20240920

author
#5 lore raw

Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()

Quoted 117 lines · Yu Zhao
Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao

With alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().

---
  include/linux/cma.h |  1 +
  mm/cma.c            | 47 ++++++++++++++++++++++++++++++---------------
  2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
index 9db877506ea8..086553fbda73 100644
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
                                      struct cma **res_cma);
  extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
                            bool no_warn);
+extern struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
  extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned long count);
  extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
mm/cma.c +29 -15
index 95d6950e177b..46feb06db8e7 100644
@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)
      spin_unlock_irq(&cma->lock);
  }
 
-/**
- * cma_alloc() - allocate pages from contiguous area
- * @cma:   Contiguous memory region for which the allocation is performed.
- * @count: Requested number of pages.
- * @align: Requested alignment of pages (in PAGE_SIZE order).
- * @no_warn: Avoid printing message about failed allocation
- *
- * This function allocates part of contiguous memory on specific
- * contiguous memory area.
- */
-struct page *cma_alloc(struct cma *cma, unsigned long count,
-                    unsigned int align, bool no_warn)
+static struct page *__cma_alloc(struct cma *cma, unsigned long count,
+                             unsigned int align, gfp_t gfp)
  {
      unsigned long mask, offset;
      unsigned long pfn = -1;
@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 
              pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
              mutex_lock(&cma_mutex);
-             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
-                                  GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp);
              mutex_unlock(&cma_mutex);
              if (ret == 0) {
                      page = pfn_to_page(pfn);
@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
                      page_kasan_tag_reset(nth_page(page, i));
      }
 
-     if (ret && !no_warn) {
+     if (ret && !(gfp & __GFP_NOWARN)) {
              pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
                                 __func__, cma->name, count, ret);
              cma_debug_show_areas(cma);
@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
      return page;
  }
 
+/**
+ * cma_alloc() - allocate pages from contiguous area
+ * @cma:   Contiguous memory region for which the allocation is performed.
+ * @count: Requested number of pages.
+ * @align: Requested alignment of pages (in PAGE_SIZE order).
+ * @no_warn: Avoid printing message about failed allocation
+ *
+ * This function allocates part of contiguous memory on specific
+ * contiguous memory area.
+ */
+struct page *cma_alloc(struct cma *cma, unsigned long count,
+                    unsigned int align, bool no_warn)
+{
+     return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+}
+
+struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
+{
+     struct page *page;
+
+     if (WARN_ON(order && !(gfp | __GFP_COMP)))
+             return NULL;
+
+     page = __cma_alloc(cma, 1 << order, order, gfp);
+
Quoted 8 lines

mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530

Quoted 1 line
mm/cma.c +1 -0
+     return page ? page_folio(page) : NULL;

We don't set large_rmappable for cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must without large_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we set large_rmappable here, and clear it in init_new_hugetlb_folio()?

I want to hear what Matthew thinks about this.

My opinion is that we don't want to couple largely rmappable (or deferred splittable) with __GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.

Quoted 6 lines

Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.

[1] https://lpc.events/event/18/sessions/184/#20240920

Also along the similar lines, Usama is trying to add PG_partially_mapped [1], which I have explicitly asked him not to introduce that flag to hugeTLB, unless there are good reasons (none ATM).

[1] https://lore.kernel.org/CAOUHufbmgwZwzUuHVvEDMqPGcsxE2hEreRZ4PhK5yz27GdK-Tw@mail.gmail.com/

#6 lore raw

Re: [PATCH mm-unstable v1 2/3] mm/cma: add cma_alloc_folio()

Quoted 123 lines · Yu Zhao
Quoted 122 lines · Yu Zhao
Quoted 99 lines · Kefeng Wang
Quoted 95 lines · Yu Zhao

With alloc_contig_range() and free_contig_range() supporting large folios, CMA can allocate and free large folios too, by cma_alloc_folio() and cma_release().

---
  include/linux/cma.h |  1 +
  mm/cma.c            | 47 ++++++++++++++++++++++++++++++---------------
  2 files changed, 33 insertions(+), 15 deletions(-)
include/linux/cma.h +1 -0
index 9db877506ea8..086553fbda73 100644
@@ -46,6 +46,7 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
                                      struct cma **res_cma);
  extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align,
                            bool no_warn);
+extern struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
  extern bool cma_pages_valid(struct cma *cma, const struct page *pages, unsigned long count);
  extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
 
mm/cma.c +29 -15
index 95d6950e177b..46feb06db8e7 100644
@@ -403,18 +403,8 @@ static void cma_debug_show_areas(struct cma *cma)
      spin_unlock_irq(&cma->lock);
  }
 
-/**
- * cma_alloc() - allocate pages from contiguous area
- * @cma:   Contiguous memory region for which the allocation is performed.
- * @count: Requested number of pages.
- * @align: Requested alignment of pages (in PAGE_SIZE order).
- * @no_warn: Avoid printing message about failed allocation
- *
- * This function allocates part of contiguous memory on specific
- * contiguous memory area.
- */
-struct page *cma_alloc(struct cma *cma, unsigned long count,
-                    unsigned int align, bool no_warn)
+static struct page *__cma_alloc(struct cma *cma, unsigned long count,
+                             unsigned int align, gfp_t gfp)
  {
      unsigned long mask, offset;
      unsigned long pfn = -1;
@@ -463,8 +453,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 
              pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
              mutex_lock(&cma_mutex);
-             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
-                                  GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+             ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp);
              mutex_unlock(&cma_mutex);
              if (ret == 0) {
                      page = pfn_to_page(pfn);
@@ -494,7 +483,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
                      page_kasan_tag_reset(nth_page(page, i));
      }
 
-     if (ret && !no_warn) {
+     if (ret && !(gfp & __GFP_NOWARN)) {
              pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
                                 __func__, cma->name, count, ret);
              cma_debug_show_areas(cma);
@@ -513,6 +502,34 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
      return page;
  }
 
+/**
+ * cma_alloc() - allocate pages from contiguous area
+ * @cma:   Contiguous memory region for which the allocation is performed.
+ * @count: Requested number of pages.
+ * @align: Requested alignment of pages (in PAGE_SIZE order).
+ * @no_warn: Avoid printing message about failed allocation
+ *
+ * This function allocates part of contiguous memory on specific
+ * contiguous memory area.
+ */
+struct page *cma_alloc(struct cma *cma, unsigned long count,
+                    unsigned int align, bool no_warn)
+{
+     return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
+}
+
+struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
+{
+     struct page *page;
+
+     if (WARN_ON(order && !(gfp | __GFP_COMP)))
+             return NULL;
+
+     page = __cma_alloc(cma, 1 << order, order, gfp);
+
Quoted 8 lines

mm/cma.c struct page *cma_alloc(struct cma *cma, unsigned long count, · line 530

Quoted 1 line
mm/cma.c +1 -0
+     return page ? page_folio(page) : NULL;

We don't set large_rmappable for cma alloc folio, which is not consistent with other folio allocation, eg folio_alloc/folio_alloc_mpol(), there is no issue for HugeTLB folio, and for HugeTLB folio must without large_rmappable, but once we use it for mTHP/THP, it need some extra handle, maybe we set large_rmappable here, and clear it in init_new_hugetlb_folio()?

I want to hear what Matthew thinks about this.

My opinion is that we don't want to couple largely rmappable (or deferred splittable) with __GFP_COMP, and for that matter, with large folios, because the former are specific to THPs whereas the latter can potentially work for most types of high order allocations.

Again, IMO, if we want to seriously answer the question of
Can we get rid of non-compound multi-page allocations? [1]
then we should start planning on decouple large rmappable from the generic folio allocation API.

Quoted 8 lines
Quoted 1 line

[1] https://lpc.events/event/18/sessions/184/#20240920

Also along the similar lines, Usama is trying to add PG_partially_mapped [1], which I have explicitly asked him not to introduce that flag to hugeTLB, unless there are good reasons (none ATM).

[1] https://lore.kernel.org/CAOUHufbmgwZwzUuHVvEDMqPGcsxE2hEreRZ4PhK5yz27GdK-Tw@mail.gmail.com/

PG_partially_mapped won't be cleared for hugeTLB in the next revision of the series as suggested by Yu. Its not there in the fix patch I posted as well in https://lore.kernel.org/all/[email protected]/

author
#7 lore raw

PATCH 3/3 mm/hugetlb: use __GFP_COMP for gigantic folios

In reply to Yu Zhao

Use __GFP_COMP for gigantic folios to greatly reduce not only the code but also the allocation and free time.

LOC (approximately): -200, +50

Allocate and free 500 1GB hugeTLB memory without HVO by:
time echo 500 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages time echo 0 >/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages

       Before  After
Alloc  ~13s    ~10s
Free   ~15s    <1s

The above magnitude generally holds for multiple x86 and arm64 CPU models.

---
 include/linux/hugetlb.h |   9 +-
 mm/hugetlb.c            | 244 ++++++++--------------------------------
 2 files changed, 50 insertions(+), 203 deletions(-)
include/linux/hugetlb.h +5 -4
index 3100a52ceb73..98c47c394b89 100644
@@ -896,10 +896,11 @@ static inline bool hugepage_movable_supported(struct hstate *h)
 /* Movability of hugepages depends on migration support. */
 static inline gfp_t htlb_alloc_mask(struct hstate *h)
 {
-	if (hugepage_movable_supported(h))
-		return GFP_HIGHUSER_MOVABLE;
-	else
-		return GFP_HIGHUSER;
+	gfp_t gfp = __GFP_COMP | __GFP_NOWARN;
+
+	gfp |= hugepage_movable_supported(h) ? GFP_HIGHUSER_MOVABLE : GFP_HIGHUSER;
+
+	return gfp;
 }
 
 static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
mm/hugetlb.c +45 -199

346 lines not on this page · raw message

Signature · 1 line
2.46.0.76.ge559c4bf1a-goog