Unify hugetlb into arch_get_unmapped_area functions

RFC PATCH In review Oscar Salvador 2 participants 11 messages Jul 10, 2024 – Jul 12, 2024

Subsystems (14)

Memory management x86 PowerPC MIPS +10 more subsystems

lore

Share & tools

lore

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #0 lore raw

cover Unify hugetlb into arch_get_unmapped_area functions

Hi all,

this is a first attempt to get rid of a fair amount of duplicated code wrt. hugetlb and *get_unmapped_area* functions.

HugeTLB registers a .get_unmapped_area function which gets called from __get_unmapped_area().
hugetlb_get_unmapped_area() is defined by a bunch of architectures and it also has a generic definition for those that do not define it. Short-long story is that there is a ton of duplicated code between specific hugetlb *_get_unmapped_area_* functions and mm-core functions, so we can do better by teaching arch_get_unmapped_area* functions how to deal with hugetlb mappings.

Note that not a lot of things need to be taught though. hugetlb_mmap_check_and_align(), that gets called for hugetlb mappings prior to call mm_get_unmapped_area_vmflags(), runs some sanity checks and aligns the addr to huge_page_size(), so we do not need to that down the road in the respective {generic,arch}_get_unmapped_area* functions.

More information can be found in the respective patches.

LTP mmapstress and hugemmap testcases were ran on arm64, powerpc64le and x86_64 without raising any issue.

Oscar Salvador (8):
mm/mmap: Teach generic_get_unmapped_area{_topdown} to handle hugetlb
mappings
arch/s390: Teach arch_get_unmapped_area{_topdown} to handle hugetlb
mappings
arch/x86: Teach arch_get_unmapped_area_vmflags to handle hugetlb
mappings
arch/sparc: Teach arch_get_unmapped_area{_topdown} to handle hugetlb
mappings
arch/powerpc: Teach book3s64 arch_get_unmapped_area{_topdown} to
handle hugetlb mappings
mm: Make hugetlb mappings go through mm_get_unmapped_area_vmflags
mm: Drop hugetlb_get_unmapped_area{_*} functions
mm: Consolidate common checks in hugetlb_mmap_check_and_align

 arch/loongarch/include/asm/hugetlb.h |   4 -
 arch/mips/include/asm/hugetlb.h      |   4 -
 arch/parisc/include/asm/hugetlb.h    |  15 ----
 arch/parisc/mm/hugetlbpage.c         |  23 ------
 arch/powerpc/mm/book3s64/slice.c     |  49 +++++++-----
 arch/s390/include/asm/hugetlb.h      |  16 ----
 arch/s390/mm/hugetlbpage.c           |  84 ---------------------
 arch/s390/mm/mmap.c                  |   8 +-
 arch/sh/include/asm/hugetlb.h        |  15 ----
 arch/sparc/kernel/sys_sparc_32.c     |  16 +++-
 arch/sparc/kernel/sys_sparc_64.c     |  36 +++++++--
 arch/sparc/mm/hugetlbpage.c          | 108 ---------------------------
 arch/x86/kernel/sys_x86_64.c         |  24 ++++--
 arch/x86/mm/hugetlbpage.c            | 100 -------------------------
 fs/hugetlbfs/inode.c                 |  92 ++---------------------
 include/asm-generic/hugetlb.h        |   7 --
 include/linux/hugetlb.h              |  21 +++---
 mm/mmap.c                            |  19 ++++-
 18 files changed, 129 insertions(+), 512 deletions(-)

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #1 lore raw

RFC PATCH 1/8 mm/mmap: Teach generic_get_unmapped_area{_topdown} to handle hugetlb mappings

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach generic_get_unmapped_area{_topdown} to handle those.
The main difference is that we set info.align_mask for huge mappings.

Signed-off-by Oscar Salvador <[email protected]>

---
 include/linux/hugetlb.h | 10 ++++++++++
 mm/mmap.c               |  4 ++++
 2 files changed, 14 insertions(+)

include/linux/hugetlb.h +10 -0

index 2b3c3a404769..1c7b0b32ff7e 100644
@@ -1029,9 +1029,19 @@ void hugetlb_unregister_node(struct node *node);
  */
 bool is_raw_hwpoison_page_in_hugepage(struct page *page);
 
+static inline unsigned long huge_page_mask_align(struct file *file)
+{
+	return PAGE_MASK & ~huge_page_mask(hstate_file(file));
+}
+
 #else	/* CONFIG_HUGETLB_PAGE */
 struct hstate {};
 
+static inline unsigned long huge_page_mask_align(struct file *file)
+{
+	return 0;
+}
+
 static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
 {
 	return NULL;

mm/mmap.c +4 -0

index 83b4682ec85c..09131b705e7b 100644
@@ -1743,6 +1743,8 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr,
 	info.length = len;
 	info.low_limit = mm->mmap_base;
 	info.high_limit = mmap_end;
+	if (filp && is_file_hugepages(filp))
+		info.align_mask = huge_page_mask_align(filp);
 	return vm_unmapped_area(&info);
 }

@@ -1791,6 +1793,8 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 	info.length = len;
 	info.low_limit = PAGE_SIZE;
 	info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
+	if (filp && is_file_hugepages(filp))
+		info.align_mask = huge_page_mask_align(filp);
 	addr = vm_unmapped_area(&info);
 
 	/*

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #2 lore raw

RFC PATCH 2/8 arch/s390: Teach arch_get_unmapped_area{_topdown} to handle hugetlb mappings

In reply to Oscar Salvador

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach arch_get_unmapped_area{_topdown} to handle those.
s390 specific hugetlb function does not set info.align_offset, so do the same here for compatibility.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/s390/mm/mmap.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

arch/s390/mm/mmap.c +6 -2

index 206756946589..e98c00df586f 100644
@@ -73,6 +73,8 @@ static inline unsigned long mmap_base(unsigned long rnd,
 
 static int get_align_mask(struct file *filp, unsigned long flags)
 {
+	if (filp && is_file_hugepages(filp))
+		return huge_page_mask_align(filp);
 	if (!(current->flags & PF_RANDOMIZE))
 		return 0;
 	if (filp || (flags & MAP_SHARED))

@@ -106,7 +108,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr,
 	info.low_limit = mm->mmap_base;
 	info.high_limit = TASK_SIZE;
 	info.align_mask = get_align_mask(filp, flags);
-	info.align_offset = pgoff << PAGE_SHIFT;
+	if (!(file && is_file_hugepages(filp))
+		info.align_offset = pgoff << PAGE_SHIFT;
 	addr = vm_unmapped_area(&info);
 	if (offset_in_page(addr))
 		return addr;

@@ -144,7 +147,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad
 	info.low_limit = PAGE_SIZE;
 	info.high_limit = mm->mmap_base;
 	info.align_mask = get_align_mask(filp, flags);
-	info.align_offset = pgoff << PAGE_SHIFT;
+	if (!(file && is_file_hugepages(filp))
+		info.align_offset = pgoff << PAGE_SHIFT;
 	addr = vm_unmapped_area(&info);
 
 	/*

Signature · 1 line

2.45.2

kernel test robot

build bot

2024-07-12 08:08 UTC #3 lore raw

Re: [RFC PATCH 2/8] arch/s390: Teach arch_get_unmapped_area{_topdown} to handle hugetlb mappings

kernel test robot noticed the following build errors:

Hi Oscar,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:

[auto build test ERROR on akpm-mm/mm-everything]
[also build test ERROR on s390/features powerpc/next powerpc/fixes deller-parisc/for-next linus/master v6.10-rc7 next-20240711] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Oscar-Salvador/mm-mmap-Teach-generic_get_unmapped_area-_topdown-to-handle-hugetlb-mappings/20240710-233828
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20240710105042.30165-3-osalvador%40suse.de
patch subject: [RFC PATCH 2/8] arch/s390: Teach arch_get_unmapped_area{_topdown} to handle hugetlb mappings
config: s390-allnoconfig (https://download.01.org/0day-ci/archive/20240712/[email protected]/config)
compiler: clang version 19.0.0git (https://github.com/llvm/llvm-project a0c6b8aef853eedaa0980f07c0a502a5a8a9740e)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240712/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from arch/s390/mm/mmap.c:13:
In file included from include/linux/mm.h:2221:
include/linux/vmstat.h:514:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]

     514 |         return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
         |                               ~~~~~~~~~~~ ^ ~~~
>> arch/s390/mm/mmap.c:76:14: error: call to undeclared function 'is_file_hugepages'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
      76 |         if (filp && is_file_hugepages(filp))
         |                     ^
>> arch/s390/mm/mmap.c:77:10: error: call to undeclared function 'huge_page_mask_align'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
      77 |                 return huge_page_mask_align(filp);
         |                        ^
   arch/s390/mm/mmap.c:111:16: error: call to undeclared function 'is_file_hugepages'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
     111 |         if (!(file && is_file_hugepages(filp))
         |                       ^
>> arch/s390/mm/mmap.c:111:8: error: use of undeclared identifier 'file'; did you mean 'filp'?
     111 |         if (!(file && is_file_hugepages(filp))
         |               ^~~~
         |               filp
   arch/s390/mm/mmap.c:85:51: note: 'filp' declared here
      85 | unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr,
         |                                                   ^
>> arch/s390/mm/mmap.c:112:3: error: expected ')'
     112 |                 info.align_offset = pgoff << PAGE_SHIFT;
         |                 ^
   arch/s390/mm/mmap.c:111:5: note: to match this '('
     111 |         if (!(file && is_file_hugepages(filp))
         |            ^
   arch/s390/mm/mmap.c:150:16: error: call to undeclared function 'is_file_hugepages'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
     150 |         if (!(file && is_file_hugepages(filp))
         |                       ^
   arch/s390/mm/mmap.c:150:8: error: use of undeclared identifier 'file'; did you mean 'filp'?
     150 |         if (!(file && is_file_hugepages(filp))
         |               ^~~~
         |               filp
   arch/s390/mm/mmap.c:121:59: note: 'filp' declared here
     121 | unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
         |                                                           ^
   arch/s390/mm/mmap.c:151:3: error: expected ')'
     151 |                 info.align_offset = pgoff << PAGE_SHIFT;
         |                 ^
   arch/s390/mm/mmap.c:150:5: note: to match this '('
     150 |         if (!(file && is_file_hugepages(filp))
         |            ^

1 warning and 8 errors generated.

vim +/is_file_hugepages +76 arch/s390/mm/mmap.c

    73	
    74	static int get_align_mask(struct file *filp, unsigned long flags)
    75	{
  > 76		if (filp && is_file_hugepages(filp))
  > 77			return huge_page_mask_align(filp);
    78		if (!(current->flags & PF_RANDOMIZE))
    79			return 0;
    80		if (filp || (flags & MAP_SHARED))
    81			return MMAP_ALIGN_MASK << PAGE_SHIFT;
    82		return 0;
    83	}
    84	
    85	unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr,
    86					     unsigned long len, unsigned long pgoff,
    87					     unsigned long flags)
    88	{
    89		struct mm_struct *mm = current->mm;
    90		struct vm_area_struct *vma;
    91		struct vm_unmapped_area_info info = {};
    92	
    93		if (len > TASK_SIZE - mmap_min_addr)
    94			return -ENOMEM;
    95	
    96		if (flags & MAP_FIXED)
    97			goto check_asce_limit;
    98	
    99		if (addr) {
   100			addr = PAGE_ALIGN(addr);
   101			vma = find_vma(mm, addr);
   102			if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
   103			    (!vma || addr + len <= vm_start_gap(vma)))
   104				goto check_asce_limit;
   105		}
   106	
   107		info.length = len;
   108		info.low_limit = mm->mmap_base;
   109		info.high_limit = TASK_SIZE;
   110		info.align_mask = get_align_mask(filp, flags);
 > 111		if (!(file && is_file_hugepages(filp))
 > 112			info.align_offset = pgoff << PAGE_SHIFT;

113 addr = vm_unmapped_area(&info);
114 if (offset_in_page(addr))
115 return addr;
116
117 check_asce_limit:
118 return check_asce_limit(mm, addr, len);
119 }
120

Signature · 2 lines

0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

Oscar Salvador

author maintainer

2024-07-12 09:05 UTC #4 lore raw

Re: [RFC PATCH 2/8] arch/s390: Teach arch_get_unmapped_area{_topdown} to handle hugetlb mappings

In reply to Oscar Salvador

Quoted 7 lines · Oscar Salvador

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach arch_get_unmapped_area{_topdown} to handle those.
s390 specific hugetlb function does not set info.align_offset, so do the same here for compatibility.
Signed-off-by Oscar Salvador <[email protected]>

I could finally grab a s390 machine and I realized I made some silly syntax mistakes here.
I am running some tests on s390 to make sure all is smooth.

Signature · 2 lines

Oscar Salvador
SUSE Labs

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #5 lore raw

RFC PATCH 3/8 arch/x86: Teach arch_get_unmapped_area_vmflags to handle hugetlb mappings

In reply to Oscar Salvador

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach arch_get_unmapped_area_{topdown_}vmflags to handle those.
x86 specific hugetlb function does not set either info.start_gap or info.align_offset so the same here for compatibility.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/x86/kernel/sys_x86_64.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

arch/x86/kernel/sys_x86_64.c +16 -8

index 01d7cd85ef97..aa7491f036a7 100644
@@ -18,6 +18,7 @@
 #include <linux/random.h>
 #include <linux/uaccess.h>
 #include <linux/elf.h>
+#include <linux/hugetlb.h>
 
 #include <asm/elf.h>
 #include <asm/ia32.h>

@@ -25,8 +26,10 @@
 /*
  * Align a virtual address to avoid aliasing in the I$ on AMD F15h.
  */
-static unsigned long get_align_mask(void)
+static unsigned long get_align_mask(struct file *filp)
 {
+	if (filp && is_file_hugepages(filp))
+		return huge_page_mask_align(filp);
 	/* handle 32- and 64-bit case with a single conditional */
 	if (va_align.flags < 0 || !(va_align.flags & (2 - mmap_is_ia32())))
 		return 0;

@@ -49,7 +52,7 @@ static unsigned long get_align_mask(void)
  */
 static unsigned long get_align_bits(void)
 {
-	return va_align.bits & get_align_mask();
+	return va_align.bits & get_align_mask(NULL);
 }
 
 static int __init control_va_addr_alignment(char *str)

@@ -148,12 +151,15 @@ arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned l
 	info.length = len;
 	info.low_limit = begin;
 	info.high_limit = end;
-	info.align_offset = pgoff << PAGE_SHIFT;
-	info.start_gap = stack_guard_placement(vm_flags);
+	if (!(filp && is_file_hugepages(filp))) {
+		info.align_offset = pgoff << PAGE_SHIFT;
+		info.start_gap = stack_guard_placement(vm_flags);
+	}
 	if (filp) {
-		info.align_mask = get_align_mask();
+		info.align_mask = get_align_mask(filp);
 		info.align_offset += get_align_bits();
 	}
+
 	return vm_unmapped_area(&info);
 }

@@ -199,7 +205,10 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0,
 		info.low_limit = PAGE_SIZE;
 
 	info.high_limit = get_mmap_base(0);
-	info.start_gap = stack_guard_placement(vm_flags);
+	if (!(filp && is_file_hugepages(filp))) {
+		info.start_gap = stack_guard_placement(vm_flags);
+		info.align_offset = pgoff << PAGE_SHIFT;
+	}
 
 	/*
 	 * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area

@@ -211,9 +220,8 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0,
 	if (addr > DEFAULT_MAP_WINDOW && !in_32bit_syscall())
 		info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;
 
-	info.align_offset = pgoff << PAGE_SHIFT;
 	if (filp) {
-		info.align_mask = get_align_mask();
+		info.align_mask = get_align_mask(filp);
 		info.align_offset += get_align_bits();
 	}
 	addr = vm_unmapped_area(&info);

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #6 lore raw

RFC PATCH 4/8 arch/sparc: Teach arch_get_unmapped_area{_topdown} to handle hugetlb mappings

In reply to Oscar Salvador

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach arch_get_unmapped_area{_topdown} to handle those.
sparc specific hugetlb function does not set info.align_offset, and does not care about adjusting the align_mask for MAP_SHARED cases, so the same here for compatibility.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/sparc/kernel/sys_sparc_32.c | 16 ++++++++++----
 arch/sparc/kernel/sys_sparc_64.c | 36 +++++++++++++++++++++++++-------
 2 files changed, 40 insertions(+), 12 deletions(-)

arch/sparc/kernel/sys_sparc_32.c +12 -4

index 08a19727795c..fc33ac48ed28 100644
@@ -42,12 +42,16 @@ SYSCALL_DEFINE0(getpagesize)
 unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags)
 {
 	struct vm_unmapped_area_info info = {};
+	bool file_hugepage = false;
+
+	if (filp && is_file_hugepages(filp))
+		file_hugepage = true;
 
 	if (flags & MAP_FIXED) {
 		/* We do not accept a shared mapping if it would violate
 		 * cache aliasing constraints.
 		 */
-		if ((flags & MAP_SHARED) &&
+		if (!file_hugepage && (flags & MAP_SHARED) &&
 		    ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)))
 			return -EINVAL;
 		return addr;

@@ -62,9 +66,13 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi
 	info.length = len;
 	info.low_limit = addr;
 	info.high_limit = TASK_SIZE;
-	info.align_mask = (flags & MAP_SHARED) ?
-		(PAGE_MASK & (SHMLBA - 1)) : 0;
-	info.align_offset = pgoff << PAGE_SHIFT;
+	if (!file_hugepage) {
+		info.align_mask = (flags & MAP_SHARED) ?
+			(PAGE_MASK & (SHMLBA - 1)) : 0;
+		info.align_offset = pgoff << PAGE_SHIFT;
+	} else {
+		info.align_mask = huge_page_mask_align(filp);
+	}
 	return vm_unmapped_area(&info);
 }

arch/sparc/kernel/sys_sparc_64.c +28 -8

index d9c3b34ca744..81d8f5467dd8 100644
@@ -87,6 +87,16 @@ static inline unsigned long COLOR_ALIGN(unsigned long addr,
 	return base + off;
 }
 
+static unsigned long get_align_mask(struct file *filp, unsigned long flags)
+{
+	if (filp && is_file_hugepages(filp))
+		return huge_page_mask_align(filp);
+	if (filp || (flags & MAP_SHARED))
+		return PAGE_MASK & (SHMLBA - 1);
+
+	return 0;
+}
+
 unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags)
 {
 	struct mm_struct *mm = current->mm;

@@ -94,12 +104,16 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi
 	unsigned long task_size = TASK_SIZE;
 	int do_color_align;
 	struct vm_unmapped_area_info info = {};
+	bool file_hugepage = false;
+
+	if (filp && is_file_hugepages(filp))
+		file_hugepage = true;
 
 	if (flags & MAP_FIXED) {
 		/* We do not accept a shared mapping if it would violate
 		 * cache aliasing constraints.
 		 */
-		if ((flags & MAP_SHARED) &&
+		if (!file_hugepage && (flags & MAP_SHARED) &&
 		    ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)))
 			return -EINVAL;
 		return addr;

@@ -111,7 +125,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi
 		return -ENOMEM;
 
 	do_color_align = 0;
-	if (filp || (flags & MAP_SHARED))
+	if ((filp || (flags & MAP_SHARED)) && !file_hugepage)
 		do_color_align = 1;
 
 	if (addr) {

@@ -129,8 +143,9 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi
 	info.length = len;
 	info.low_limit = TASK_UNMAPPED_BASE;
 	info.high_limit = min(task_size, VA_EXCLUDE_START);
-	info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
-	info.align_offset = pgoff << PAGE_SHIFT;
+	info.align_mask = get_align_mask(filp, flags);
+	if (!file_hugepage)
+		info.align_offset = pgoff << PAGE_SHIFT;
 	addr = vm_unmapped_area(&info);
 
 	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {

@@ -154,15 +169,19 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
 	unsigned long addr = addr0;
 	int do_color_align;
 	struct vm_unmapped_area_info info = {};
+	bool file_hugepage = false
 
 	/* This should only ever run for 32-bit processes.  */
 	BUG_ON(!test_thread_flag(TIF_32BIT));
 
+	if (filp && is_file_hugepages(filp))
+		file_hugepage = true;
+
 	if (flags & MAP_FIXED) {
 		/* We do not accept a shared mapping if it would violate
 		 * cache aliasing constraints.
 		 */
-		if ((flags & MAP_SHARED) &&
+		if (!file_hugepage && (flags & MAP_SHARED) &&
 		    ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)))
 			return -EINVAL;
 		return addr;

@@ -172,7 +191,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
 		return -ENOMEM;
 
 	do_color_align = 0;
-	if (filp || (flags & MAP_SHARED))
+	if ((filp || (flags & MAP_SHARED)) && !file_hugepage)
 		do_color_align = 1;
 
 	/* requesting a specific address */

@@ -192,8 +211,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
 	info.length = len;
 	info.low_limit = PAGE_SIZE;
 	info.high_limit = mm->mmap_base;
-	info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0;
-	info.align_offset = pgoff << PAGE_SHIFT;
+	info.align_mask = get_align_mask(filp, flags);
+	if (!file_hugepage)
+		info.align_offset = pgoff << PAGE_SHIFT;
 	addr = vm_unmapped_area(&info);
 
 	/*

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #7 lore raw

RFC PATCH 5/8 arch/powerpc: Teach book3s64 arch_get_unmapped_area{_topdown} to handle hugetlb mappings

In reply to Oscar Salvador

We want to stop special casing hugetlb mappings and make them go through generic channels, so teach arch_get_unmapped_area{_topdown} to handle those.
Reshuffle file_to_psize() definition so arch_get_unmapped_area{_topdown} can make use of it.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/powerpc/mm/book3s64/slice.c | 39 ++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 10 deletions(-)

arch/powerpc/mm/book3s64/slice.c +29 ~~-10~~

index ef3ce37f1bb3..6914b8de627c 100644
@@ -633,17 +633,36 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
 }
 EXPORT_SYMBOL_GPL(slice_get_unmapped_area);
 
+#ifdef CONFIG_HUGETLB_PAGE
+static int file_to_psize(struct file *file)
+{
+	struct hstate *hstate = hstate_file(file);
+	return shift_to_mmu_psize(huge_page_shift(hstate));
+}
+#else
+static int file_to_psize(struct file *file)
+{
+	return 0;
+}
+#endif
+
 unsigned long arch_get_unmapped_area(struct file *filp,
 				     unsigned long addr,
 				     unsigned long len,
 				     unsigned long pgoff,
 				     unsigned long flags)
 {
+	unsigned int psize;
+
 	if (radix_enabled())
 		return generic_get_unmapped_area(filp, addr, len, pgoff, flags);
 
-	return slice_get_unmapped_area(addr, len, flags,
-				       mm_ctx_user_psize(&current->mm->context), 0);
+	if (filp && is_file_hugepages(filp))
+		psize = file_to_psize(filp);
+	else
+		psize = mm_ctx_user_psize(&current->mm->context);
+
+	return slice_get_unmapped_area(addr, len, flags, psize, 0);
 }
 
 unsigned long arch_get_unmapped_area_topdown(struct file *filp,

@@ -652,11 +671,17 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp,
 					     const unsigned long pgoff,
 					     const unsigned long flags)
 {
+	unsigned int psize;
+
 	if (radix_enabled())
 		return generic_get_unmapped_area_topdown(filp, addr0, len, pgoff, flags);
 
-	return slice_get_unmapped_area(addr0, len, flags,
-				       mm_ctx_user_psize(&current->mm->context), 1);
+	if (filp && is_file_hugepages(filp))
+		psize = file_to_psize(filp);
+	else
+		psize = mm_ctx_user_psize(&current->mm->context);
+
+	return slice_get_unmapped_area(addr0, len, flags, psize, 1);
 }
 
 unsigned int notrace get_slice_psize(struct mm_struct *mm, unsigned long addr)

@@ -787,12 +812,6 @@ unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 	return 1UL << mmu_psize_to_shift(get_slice_psize(vma->vm_mm, vma->vm_start));
 }
 
-static int file_to_psize(struct file *file)
-{
-	struct hstate *hstate = hstate_file(file);
-	return shift_to_mmu_psize(huge_page_shift(hstate));
-}
-
 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 					unsigned long len, unsigned long pgoff,
 					unsigned long flags)

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #8 lore raw

RFC PATCH 6/8 mm: Make hugetlb mappings go through mm_get_unmapped_area_vmflags

In reply to Oscar Salvador

Hugetlb mappings will no longer be special cased but rather go through the generic mm_get_unmapped_area_vmflags function.
For that to happen, let us remove the .get_unmapped_area from hugetlbfs_file_operations struct, and hint __get_unmapped_area that it should not send hugetlb mappings through thp_get_unmapped_area_vmflags but through mm_get_unmapped_area_vmflags.

Create also a function called hugetlb_mmap_check_and_align() where a couple of safety checks are being done and the addr is aligned to the huge page size.
Otherwise we will have to do this in every single function, which duplicates quite a lot of code.

Signed-off-by Oscar Salvador <[email protected]>

---
 fs/hugetlbfs/inode.c    | 22 ++++++++++++++--------
 include/linux/hugetlb.h |  8 +++-----
 mm/mmap.c               | 15 ++++++++++++++-
 3 files changed, 31 insertions(+), 14 deletions(-)

fs/hugetlbfs/inode.c +14 -8

index 412f295acebe..b2d7fcecdb15 100644
@@ -257,15 +257,22 @@ generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 			pgoff, flags);
 }
 
-#ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-static unsigned long
-hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-			  unsigned long len, unsigned long pgoff,
-			  unsigned long flags)
+unsigned long
+hugetlb_mmap_check_and_align(struct file *file, unsigned long addr,
+			     unsigned long len, unsigned long flags)
 {
-	return generic_hugetlb_get_unmapped_area(file, addr, len, pgoff, flags);
+	unsigned long addr0 = 0;
+	struct hstate *h = hstate_file(file);
+
+	if (len & ~huge_page_mask(h))
+		return -EINVAL;
+	if ((flags & MAP_FIXED) && prepare_hugepage_range(file, addr, len))
+		return -EINVAL;
+	if (addr)
+		addr0 = ALIGN(addr, huge_page_size(h));
+
+	return addr0;
 }
-#endif
 
 /*
  * Someone wants to read @bytes from a HWPOISON hugetlb @page from @offset.

@@ -1302,7 +1309,6 @@ static const struct file_operations hugetlbfs_file_operations = {
 	.read_iter		= hugetlbfs_read_iter,
 	.mmap			= hugetlbfs_file_mmap,
 	.fsync			= noop_fsync,
-	.get_unmapped_area	= hugetlb_get_unmapped_area,
 	.llseek			= default_llseek,
 	.fallocate		= hugetlbfs_fallocate,
 	.fop_flags		= FOP_HUGE_PAGES,

include/linux/hugetlb.h +3 -5

index 1c7b0b32ff7e..9183ef95dfb6 100644
@@ -555,11 +555,9 @@ static inline struct hstate *hstate_inode(struct inode *i)
 }
 #endif /* !CONFIG_HUGETLBFS */
 
-#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-					unsigned long len, unsigned long pgoff,
-					unsigned long flags);
-#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
+unsigned long
+hugetlb_mmap_check_and_align(struct file *file, unsigned long addr,
+			     unsigned long len, unsigned long flags);
 
 unsigned long
 generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr,

mm/mmap.c +14 -1

index 09131b705e7b..8130b25b8cf5 100644
@@ -1860,6 +1860,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 				  unsigned long, unsigned long, unsigned long)
 				  = NULL;
 
+	bool is_hugetlb = false;
 	unsigned long error = arch_mmap_check(addr, len, flags);
 	if (error)
 		return error;

@@ -1868,6 +1869,9 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 	if (len > TASK_SIZE)
 		return -ENOMEM;
 
+	if (file && is_file_hugepages(file))
+		is_hugetlb = true;
+
 	if (file) {
 		if (file->f_op->get_unmapped_area)
 			get_area = file->f_op->get_unmapped_area;

@@ -1885,11 +1889,20 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 
 	if (get_area) {
 		addr = get_area(file, addr, len, pgoff, flags);
-	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && !is_hugetlb) {
 		/* Ensures that larger anonymous mappings are THP aligned. */
 		addr = thp_get_unmapped_area_vmflags(file, addr, len,
 						     pgoff, flags, vm_flags);
 	} else {
+		/*
+		 * Consolidate hugepages checks in one place, and also align addr
+		 * to hugepage size.
+		 */
+		if (is_hugetlb) {
+			addr = hugetlb_mmap_check_and_align(file, addr, len, flags);
+			if (IS_ERR_VALUE(addr))
+				return addr;
+		}
 		addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len,
 						    pgoff, flags, vm_flags);
 	}

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #9 lore raw

RFC PATCH 7/8 mm: Drop hugetlb_get_unmapped_area{_*} functions

In reply to Oscar Salvador

Hugetlb mappings are now handled through normal channels just like any other mapping, so we no longer need hugetlb_get_unmapped_area* specific functions.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/parisc/mm/hugetlbpage.c     |  23 -------
 arch/powerpc/mm/book3s64/slice.c |  10 ---
 arch/s390/mm/hugetlbpage.c       |  84 ------------------------
 arch/sparc/mm/hugetlbpage.c      | 108 -------------------------------
 arch/x86/mm/hugetlbpage.c        | 100 ----------------------------
 fs/hugetlbfs/inode.c             |  86 ------------------------
 include/linux/hugetlb.h          |   5 --
 7 files changed, 416 deletions(-)

arch/parisc/mm/hugetlbpage.c +0 ~~-23~~

index 0356199bd9e7..69c463d6ffd6 100644
@@ -21,29 +21,6 @@
 #include <asm/mmu_context.h>
 
 
-unsigned long
-hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-		unsigned long len, unsigned long pgoff, unsigned long flags)
-{
-	struct hstate *h = hstate_file(file);
-
-	if (len & ~huge_page_mask(h))
-		return -EINVAL;
-	if (len > TASK_SIZE)
-		return -ENOMEM;
-
-	if (flags & MAP_FIXED)
-		if (prepare_hugepage_range(file, addr, len))
-			return -EINVAL;
-
-	if (addr)
-		addr = ALIGN(addr, huge_page_size(h));
-
-	/* we need to make sure the colouring is OK */
-	return arch_get_unmapped_area(file, addr, len, pgoff, flags);
-}
-
-
 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
 			unsigned long addr, unsigned long sz)
 {

arch/powerpc/mm/book3s64/slice.c +0 ~~-10~~

index 6914b8de627c..6ce16bc330a2 100644
@@ -811,14 +811,4 @@ unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 
 	return 1UL << mmu_psize_to_shift(get_slice_psize(vma->vm_mm, vma->vm_start));
 }
-
-unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-					unsigned long len, unsigned long pgoff,
-					unsigned long flags)
-{
-	if (radix_enabled())
-		return generic_hugetlb_get_unmapped_area(file, addr, len, pgoff, flags);
-
-	return slice_get_unmapped_area(addr, len, flags, file_to_psize(file), 1);
-}
 #endif

arch/s390/mm/hugetlbpage.c +0 ~~-84~~

87 lines not on this page · raw message

arch/sparc/mm/hugetlbpage.c +0 ~~-108~~

114 lines not on this page · raw message

arch/x86/mm/hugetlbpage.c +0 ~~-100~~

107 lines not on this page · raw message

fs/hugetlbfs/inode.c +0 ~~-86~~

92 lines not on this page · raw message

include/linux/hugetlb.h +0 -5

index 9183ef95dfb6..a83a2b331bab 100644
@@ -559,11 +559,6 @@ unsigned long
 hugetlb_mmap_check_and_align(struct file *file, unsigned long addr,
 			     unsigned long len, unsigned long flags);
 
-unsigned long
-generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
-				  unsigned long len, unsigned long pgoff,
-				  unsigned long flags);
-
 /*
  * huegtlb page specific state flags.  These flags are located in page.private
  * of the hugetlb head page.  Functions created via the below macros should be

Signature · 1 line

2.45.2

Oscar Salvador

author maintainer

2024-07-10 10:51 UTC #10 lore raw

RFC PATCH 8/8 mm: Consolidate common checks in hugetlb_mmap_check_and_align

In reply to Oscar Salvador

prepare_hugepage_range() performs almost the same checks for all architectures that define it, with the exception of mips and loongarch that also check for overflows.
The rest checks for the addr and len to be properly aligned, so we can move that to the generic hugetlb_mmap_check_and_align() function and get rid of a fair amount of duplicated code.

Signed-off-by Oscar Salvador <[email protected]>

---
 arch/loongarch/include/asm/hugetlb.h |  4 ----
 arch/mips/include/asm/hugetlb.h      |  4 ----
 arch/parisc/include/asm/hugetlb.h    | 15 ---------------
 arch/s390/include/asm/hugetlb.h      | 16 ----------------
 arch/sh/include/asm/hugetlb.h        | 15 ---------------
 fs/hugetlbfs/inode.c                 |  8 ++++++--
 include/asm-generic/hugetlb.h        |  7 -------
 7 files changed, 6 insertions(+), 63 deletions(-)

arch/loongarch/include/asm/hugetlb.h +0 -4

index aa44b3fe43dd..107566c98938 100644
@@ -18,10 +18,6 @@ static inline int prepare_hugepage_range(struct file *file,
 	unsigned long task_size = STACK_TOP;
 	struct hstate *h = hstate_file(file);
 
-	if (len & ~huge_page_mask(h))
-		return -EINVAL;
-	if (addr & ~huge_page_mask(h))
-		return -EINVAL;
 	if (len > task_size)
 		return -ENOMEM;
 	if (task_size - len < addr)

arch/mips/include/asm/hugetlb.h +0 -4

index fd69c8808554..6a63d82a8ab3 100644
@@ -19,10 +19,6 @@ static inline int prepare_hugepage_range(struct file *file,
 	unsigned long task_size = STACK_TOP;
 	struct hstate *h = hstate_file(file);
 
-	if (len & ~huge_page_mask(h))
-		return -EINVAL;
-	if (addr & ~huge_page_mask(h))
-		return -EINVAL;
 	if (len > task_size)
 		return -ENOMEM;
 	if (task_size - len < addr)

arch/parisc/include/asm/hugetlb.h +0 ~~-15~~

index 72daacc472a0..5b3a5429f71b 100644
@@ -12,21 +12,6 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep);
 
-/*
- * If the arch doesn't supply something else, assume that hugepage
- * size aligned regions are ok without further preparation.
- */
-#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE
-static inline int prepare_hugepage_range(struct file *file,
-			unsigned long addr, unsigned long len)
-{
-	if (len & ~HPAGE_MASK)
-		return -EINVAL;
-	if (addr & ~HPAGE_MASK)
-		return -EINVAL;
-	return 0;
-}
-
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
 					  unsigned long addr, pte_t *ptep)

arch/s390/include/asm/hugetlb.h +0 ~~-16~~

index ce5f4fe8be4d..38a10b47dc99 100644
@@ -23,22 +23,6 @@ pte_t huge_ptep_get(pte_t *ptep);
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 			      unsigned long addr, pte_t *ptep);
 
-/*
- * If the arch doesn't supply something else, assume that hugepage
- * size aligned regions are ok without further preparation.
- */
-static inline int prepare_hugepage_range(struct file *file,
-			unsigned long addr, unsigned long len)
-{
-	struct hstate *h = hstate_file(file);
-
-	if (len & ~huge_page_mask(h))
-		return -EINVAL;
-	if (addr & ~huge_page_mask(h))
-		return -EINVAL;
-	return 0;
-}
-
 static inline void arch_clear_hugetlb_flags(struct folio *folio)
 {
 	clear_bit(PG_arch_1, &folio->flags);

arch/sh/include/asm/hugetlb.h +0 ~~-15~~

index 75028bd568ba..4a92e6e4d627 100644
@@ -5,21 +5,6 @@
 #include <asm/cacheflush.h>
 #include <asm/page.h>
 
-/*
- * If the arch doesn't supply something else, assume that hugepage
- * size aligned regions are ok without further preparation.
- */
-#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE
-static inline int prepare_hugepage_range(struct file *file,
-			unsigned long addr, unsigned long len)
-{
-	if (len & ~HPAGE_MASK)
-		return -EINVAL;
-	if (addr & ~HPAGE_MASK)
-		return -EINVAL;
-	return 0;
-}
-
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
 					  unsigned long addr, pte_t *ptep)

fs/hugetlbfs/inode.c +6 -2

index 722e60327615..82ec499feb41 100644
@@ -180,8 +180,12 @@ hugetlb_mmap_check_and_align(struct file *file, unsigned long addr,
 
 	if (len & ~huge_page_mask(h))
 		return -EINVAL;
-	if ((flags & MAP_FIXED) && prepare_hugepage_range(file, addr, len))
-		return -EINVAL;
+	if (flags & MAP_FIXED) {
+		if (addr & ~huge_page_mask(h))
+			return -EINVAL;
+		if (prepare_hugepage_range(file, addr, len))
+			return -EINVAL;
+	}
 	if (addr)
 		addr0 = ALIGN(addr, huge_page_size(h));

include/asm-generic/hugetlb.h +0 -7

index 6dcf4d576970..d522581441e5 100644
@@ -115,13 +115,6 @@ static inline int huge_pte_none_mostly(pte_t pte)
 static inline int prepare_hugepage_range(struct file *file,
 		unsigned long addr, unsigned long len)
 {
-	struct hstate *h = hstate_file(file);
-
-	if (len & ~huge_page_mask(h))
-		return -EINVAL;
-	if (addr & ~huge_page_mask(h))
-		return -EINVAL;
-
 	return 0;
 }
 #endif

Signature · 1 line

2.45.2