Skip to contentSkip to conversation

mm/arm64: re-enable HVO

RFC PATCH In review Yu Zhao 4 participants 9 messages Aug 6, 2024 – Aug 8, 2024

Subsystems (3)
lore
Share & tools
lore

Yu Zhao

author
#0 lore raw

cover mm/arm64: re-enable HVO

This series presents one of the previously discussed approaches to re-enable HugeTLB Vmemmap Optimization (HVO) on arm64. HVO was disabled by commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the following reason:

This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.

Other approaches that have been discussed include:
A. Handle kernel PF while doing BBM [1],
B. Use stop_machine() while doing BBM [2], and,
C. Enable FEAT_BBM level 2 and keep the memory contents at the old
and new output addresses unchanged to avoid BBM (D8.16.1-2) [3].

A quick comparison between this approach (D) and the above approaches:
--+------------------------------+-----------------------------+
| Pro | Con |
--+------------------------------+-----------------------------+

  A | Low latency, h/w independent | Predictability concerns [4] |
  B | Predictable, h/w independent | High latency                |
  C | Predictable, low latency     | H/w dependent, complex      |
  D | Predictable, h/w independent | Medium latency              |

--+------------------------------+-----------------------------+

[1] https://lore.kernel.org/[email protected]/
[2] https://lore.kernel.org/[email protected]/
[3] https://lore.kernel.org/[email protected]/
[4] https://lore.kernel.org/20240326125409.GA9552@willie-the-truck/

Nanyong Sun (2):
mm: HVO: introduce helper function to update and flush pgtable
arm64: mm: Re-enable OPTIMIZE_HUGETLB_VMEMMAP

Yu Zhao (2):
arm64: use IPIs to pause/resume remote CPUs
arm64: pause remote CPUs to update vmemmap

 arch/arm64/Kconfig               |   1 +
 arch/arm64/include/asm/pgalloc.h |  55 ++++++++++++++++
 arch/arm64/include/asm/smp.h     |   3 +
 arch/arm64/kernel/smp.c          | 110 +++++++++++++++++++++++++++++++
 mm/hugetlb_vmemmap.c             |  69 +++++++++++++++----
 5 files changed, 226 insertions(+), 12 deletions(-)

base-commit: de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed

Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
author
#1 lore raw

RFC PATCH 1/4 mm: HVO: introduce helper function to update and flush pgtable

From: Nanyong Sun <[email protected]>

Add pmd/pte update and tlb flush helper function to update page table. This refactoring patch is designed to facilitate each architecture to implement its own special logic in preparation for the arm64 architecture to follow the necessary break-before-make sequence when updating page tables.

---
 mm/hugetlb_vmemmap.c | 55 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 12 deletions(-)
mm/hugetlb_vmemmap.c +43 -12
index 829112b0a914..2dd92e58f304 100644
@@ -46,6 +46,37 @@ struct vmemmap_remap_walk {
 	unsigned long		flags;
 };
 
+#ifndef vmemmap_update_pmd
+static inline void vmemmap_update_pmd(unsigned long addr,
+				      pmd_t *pmdp, pte_t *ptep)
+{
+	pmd_populate_kernel(&init_mm, pmdp, ptep);
+}
+#endif
+
+#ifndef vmemmap_update_pte
+static inline void vmemmap_update_pte(unsigned long addr,
+				      pte_t *ptep, pte_t pte)
+{
+	set_pte_at(&init_mm, addr, ptep, pte);
+}
+#endif
+
+#ifndef vmemmap_flush_tlb_all
+static inline void vmemmap_flush_tlb_all(void)
+{
+	flush_tlb_all();
+}
+#endif
+
+#ifndef vmemmap_flush_tlb_range
+static inline void vmemmap_flush_tlb_range(unsigned long start,
+					   unsigned long end)
+{
+	flush_tlb_kernel_range(start, end);
+}
+#endif
+
 static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
 			     struct vmemmap_remap_walk *walk)
 {
@@ -81,9 +112,9 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
 
 		/* Make pte visible before pmd. See comment in pmd_install(). */
 		smp_wmb();
-		pmd_populate_kernel(&init_mm, pmd, pgtable);
+		vmemmap_update_pmd(start, pmd, pgtable);
 		if (!(walk->flags & VMEMMAP_SPLIT_NO_TLB_FLUSH))
-			flush_tlb_kernel_range(start, start + PMD_SIZE);
+			vmemmap_flush_tlb_range(start, start + PMD_SIZE);
 	} else {
 		pte_free_kernel(&init_mm, pgtable);
 	}
@@ -171,7 +202,7 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
 		return ret;
 
 	if (walk->remap_pte && !(walk->flags & VMEMMAP_REMAP_NO_TLB_FLUSH))
-		flush_tlb_kernel_range(start, end);
+		vmemmap_flush_tlb_range(start, end);
 
 	return 0;
 }
@@ -220,15 +251,15 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
 
 		/*
 		 * Makes sure that preceding stores to the page contents from
-		 * vmemmap_remap_free() become visible before the set_pte_at()
-		 * write.
+		 * vmemmap_remap_free() become visible before the
+		 * vmemmap_update_pte() write.
 		 */
 		smp_wmb();
 	}
 
 	entry = mk_pte(walk->reuse_page, pgprot);
 	list_add(&page->lru, walk->vmemmap_pages);
-	set_pte_at(&init_mm, addr, pte, entry);
+	vmemmap_update_pte(addr, pte, entry);
 }
 
 /*
@@ -267,10 +298,10 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
 
 	/*
 	 * Makes sure that preceding stores to the page contents become visible
-	 * before the set_pte_at() write.
+	 * before the vmemmap_update_pte() write.
 	 */
 	smp_wmb();
-	set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
+	vmemmap_update_pte(addr, pte, mk_pte(page, pgprot));
 }
 
 /**
@@ -536,7 +567,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
 	}
 
 	if (restored)
-		flush_tlb_all();
+		vmemmap_flush_tlb_all();
 	if (!ret)
 		ret = restored;
 	return ret;
@@ -664,7 +695,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
 			break;
 	}
 
-	flush_tlb_all();
+	vmemmap_flush_tlb_all();
 
 	/* avoid writes from page_ref_add_unless() while folding vmemmap */
 	synchronize_rcu();
@@ -684,7 +715,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
 		 * allowing more vmemmap remaps to occur.
 		 */
 		if (ret == -ENOMEM && !list_empty(&vmemmap_pages)) {
-			flush_tlb_all();
+			vmemmap_flush_tlb_all();
 			free_vmemmap_page_list(&vmemmap_pages);
 			INIT_LIST_HEAD(&vmemmap_pages);
 			__hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages,
@@ -692,7 +723,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
 		}
 	}
 
-	flush_tlb_all();
+	vmemmap_flush_tlb_all();
 	free_vmemmap_page_list(&vmemmap_pages);
 }
 
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
author
#2 lore raw

RFC PATCH 2/4 arm64: use IPIs to pause/resume remote CPUs

In reply to Yu Zhao

Use pseudo-NMI IPIs to pause remote CPUs for a short period of time, and then reliably resume them when the local CPU exits critical sections that preclude the execution of remote CPUs.

A typical example of such critical sections is BBM on kernel PTEs. HugeTLB Vmemmap Optimization (HVO) on arm64 was disabled by commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the folllowing reason:

This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.

Supporting BBM on kernel PTEs is one of the approaches that can potentially make arm64 support HVO.

---
 arch/arm64/include/asm/smp.h |   3 +
 arch/arm64/kernel/smp.c      | 110 +++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
arch/arm64/include/asm/smp.h +3 -0
index 2510eec026f7..cffb0cfed961 100644
@@ -133,6 +133,9 @@ bool cpus_are_stuck_in_kernel(void);
 extern void crash_smp_send_stop(void);
 extern bool smp_crash_stop_failed(void);
 
+void pause_remote_cpus(void);
+void resume_remote_cpus(void);
+
 #endif /* ifndef __ASSEMBLY__ */
 
 #endif /* ifndef __ASM_SMP_H */
arch/arm64/kernel/smp.c +110 -0

173 lines not on this page · raw message

Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
#3 lore raw

Re: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs

[...]

Quoted 12 lines
arch/arm64/kernel/smp.c +11 -0
+
+void resume_remote_cpus(void)
+{
+	cpumask_t cpus_to_resume;
+
+	lockdep_assert_cpus_held();
+	lockdep_assert_preemption_disabled();
+
+	cpumask_copy(&cpus_to_resume, cpu_online_mask);
+	cpumask_clear_cpu(smp_processor_id(), &cpus_to_resume);
+

arch/arm64/kernel/smp.c void __noreturn panic_smp_self_stop(void) · line 979

Quoted 10 lines
arch/arm64/kernel/smp.c +10 -0
+	spin_lock(&cpu_pause_lock);
+
+	cpumask_setall(&resumed_cpus);
+	/* A typical example for sleep and wake-up functions. */
+	smp_mb();
+	while (cpumask_intersects(&cpus_to_resume, &paused_cpus)) {
+		sev();
+		cpu_relax();
+		barrier();
+	}

I'm curious, is there a fundamental reason why we wait for paused CPUs to actually start running, or is it simply easier to get the implementation race-free, in particular when we have two pause_remote_cpus() calls shortly after each other and another remote CPU might still be on its way out of pause_local_cpu() from the first pause.

Signature · 3 lines
Cheers,

David / dhildenb
#4 lore raw

Re: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs

In reply to Yu Zhao
kernel test robot noticed the following build warnings:

Hi Yu,

[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:

[auto build test WARNING on de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed]

url:    https://github.com/intel-lab-lkp/linux/commits/Yu-Zhao/mm-HVO-introduce-helper-function-to-update-and-flush-pgtable/20240806-104445
base:   de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed
patch link:    https://lore.kernel.org/r/20240806022114.3320543-3-yuzhao%40google.com
patch subject: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs
config: arm64-allnoconfig (https://download.01.org/0day-ci/archive/20240808/[email protected]/config)
compiler: aarch64-linux-gcc (GCC) 14.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240808/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> arch/arm64/kernel/smp.c:988:24: warning: 'ipi_cpu_crash_stop' defined but not used [-Wunused-function]

988 | static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
| ^~~~~~~~~~~~~~~~~~

vim +/ipi_cpu_crash_stop +988 arch/arm64/kernel/smp.c

78fd584cdec0518 AKASHI Takahiro 2017-04-03 987

5ab6876c7843db5 Josh Poimboeuf  2023-04-12  @988  static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
78fd584cdec0518 AKASHI Takahiro 2017-04-03   989  {
78fd584cdec0518 AKASHI Takahiro 2017-04-03   990  #ifdef CONFIG_KEXEC_CORE
78fd584cdec0518 AKASHI Takahiro 2017-04-03   991  	crash_save_cpu(regs, cpu);
78fd584cdec0518 AKASHI Takahiro 2017-04-03   992  
78fd584cdec0518 AKASHI Takahiro 2017-04-03   993  	atomic_dec(&waiting_for_crash_ipi);
78fd584cdec0518 AKASHI Takahiro 2017-04-03   994  
78fd584cdec0518 AKASHI Takahiro 2017-04-03   995  	local_irq_disable();
f5df26961853d68 James Morse     2018-01-08   996  	sdei_mask_local_cpu();
78fd584cdec0518 AKASHI Takahiro 2017-04-03   997  
de58ed5e16e62f3 Gavin Shan      2020-03-19   998  	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
de58ed5e16e62f3 Gavin Shan      2020-03-19   999  		__cpu_try_die(cpu);
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1000  
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1001  	/* just in case */
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1002  	cpu_park_loop();
5ab6876c7843db5 Josh Poimboeuf  2023-04-12  1003  #else
5ab6876c7843db5 Josh Poimboeuf  2023-04-12  1004  	BUG();
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1005  #endif
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1006  }
78fd584cdec0518 AKASHI Takahiro 2017-04-03  1007  
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#5 lore raw

Re: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs

In reply to Yu Zhao

Hi,

Quoted 15 lines · Yu Zhao

Use pseudo-NMI IPIs to pause remote CPUs for a short period of time, and then reliably resume them when the local CPU exits critical sections that preclude the execution of remote CPUs.

A typical example of such critical sections is BBM on kernel PTEs. HugeTLB Vmemmap Optimization (HVO) on arm64 was disabled by commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the folllowing reason:

This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.

Quoted 8 lines

Supporting BBM on kernel PTEs is one of the approaches that can potentially make arm64 support HVO.

---
 arch/arm64/include/asm/smp.h |   3 +
 arch/arm64/kernel/smp.c      | 110 +++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)

I'm a bit curious how your approach is reliable / performant in all cases. As far as I understand it:

1. Patch #4 in your series unconditionally turns on "ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP" for arm64.

2. In order for it to work reliably, you need the "pause all CPUs" functionality introduced in this patch.

3. In order for the "pause all CPUs" functionality to be performant you need NMI or, at least, pseudo-NMI to be used to pause all CPUs.

4. Even when you configure the kernel for pseudo-NMI it's not 100% guaranteed that pseudo-NMI will be turned on. Specifically:

4a) There's an extra kernel command line parameter you need to actually enable pseudo-NMI. We can debate about the inability to turn on pseudo-NMI without the command line parameter, but at the moment it's there because pseudo-NMI has some performance implications. Apparently these performance implications are more non-trivial on some early arm64 CPUs.

4b) Even if we changed it so that the command-line parameter wasn't needed, there are still some boards out there that are known not to be able to enable pseudo-NMI. There are certainly some Mediatek Chromebooks that have a BIOS bug making pseudo-NMI unreliable. See the mediatek,broken-save-restore-fw device tree property. ...and even if you ignore the Mediatek Chromebooks, there's at least one more system I know of that's broken with pseudo-NMI. Since you're at Google, you could look at b/308278090 for details but the quick summary is that some devices running a TEE are hanging when pseudo NMI is enabled. ...and, even if that's fixed, it feels somewhat likely that there are other systems where pseudo-NMI won't be usable.

Unless I'm misunderstanding, it feels like anything you have that relies on NMI/pseudo-NMI needs to fall back safely/reliably if NMI/pseudo-NMI isn't there.

Quoted 22 lines
arch/arm64/include/asm/smp.h +3 -0
index 2510eec026f7..cffb0cfed961 100644
@@ -133,6 +133,9 @@ bool cpus_are_stuck_in_kernel(void);
 extern void crash_smp_send_stop(void);
 extern bool smp_crash_stop_failed(void);
 
+void pause_remote_cpus(void);
+void resume_remote_cpus(void);
+
 #endif /* ifndef __ASSEMBLY__ */
 
 #endif /* ifndef __ASM_SMP_H */
arch/arm64/kernel/smp.c +0 -0
index 5e18fbcee9a2..aa80266e5c9d 100644
@@ -68,16 +68,25 @@ enum ipi_msg_type {
        IPI_RESCHEDULE,
        IPI_CALL_FUNC,
        IPI_CPU_STOP,

arch/arm64/kernel/smp.c enum ipi_msg_type { · line 80

Quoted 10 lines
arch/arm64/kernel/smp.c +7 -0
+       IPI_CPU_PAUSE,
+#ifdef CONFIG_KEXEC_CORE
        IPI_CPU_CRASH_STOP,
+#endif
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
        IPI_TIMER,
+#endif
+#ifdef CONFIG_IRQ_WORK
        IPI_IRQ_WORK,
+#endif

I assume all these "ifdefs" are there because this adds up to more than 8 IPIs. That means that someone wouldn't be able to enable all of these things, right? Feels like we'd want to solve this before landing things. In the least it would be good if this built upon:

https://lore.kernel.org/r/20240625160718.v2.1.Id4817adef610302554b8aa42b090d57270dc119c@changeid/

...and then maybe we could figure out if there are other ways to consolidate NMIs. Previously, for instance, we had the "KGDB" and "backtrace" IPIs combined into one but we split them upon review feedback. If necessary they would probably be easy to re-combine.

author
#6 lore raw

RFC PATCH 3/4 arm64: pause remote CPUs to update vmemmap

In reply to Yu Zhao

Pause remote CPUs so that the local CPU can follow the proper BBM sequence to safely update the vmemmap mapping struct page areas.

While updating the vmemmap, it is guaranteed that neither the local CPU nor the remote ones will access the struct page area being updated, and therefore they will not trigger kernel PFs.

---
 arch/arm64/include/asm/pgalloc.h | 55 ++++++++++++++++++++++++++++++++
 mm/hugetlb_vmemmap.c             | 14 ++++++++
 2 files changed, 69 insertions(+)
arch/arm64/include/asm/pgalloc.h +55 -0
index 8ff5f2a2579e..1af1aa34a351 100644
@@ -12,6 +12,7 @@
 #include <asm/processor.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
+#include <asm/cpu.h>
 
 #define __HAVE_ARCH_PGD_FREE
 #define __HAVE_ARCH_PUD_FREE
@@ -137,4 +138,58 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
 	__pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE | PMD_TABLE_PXN);
 }
 
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+
+#define vmemmap_update_lock vmemmap_update_lock
+static inline void vmemmap_update_lock(void)
+{
+	cpus_read_lock();
+}
+
+#define vmemmap_update_unlock vmemmap_update_unlock
+static inline void vmemmap_update_unlock(void)
+{
+	cpus_read_unlock();
+}
+
+#define vmemmap_update_pte vmemmap_update_pte
+static inline void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte)
+{
+	preempt_disable();
+	pause_remote_cpus();
+
+	pte_clear(&init_mm, addr, ptep);
+	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+	set_pte_at(&init_mm, addr, ptep, pte);
+
+	resume_remote_cpus();
+	preempt_enable();
+}
+
+#define vmemmap_update_pmd vmemmap_update_pmd
+static inline void vmemmap_update_pmd(unsigned long addr, pmd_t *pmdp, pte_t *ptep)
+{
+	preempt_disable();
+	pause_remote_cpus();
+
+	pmd_clear(pmdp);
+	flush_tlb_kernel_range(addr, addr + PMD_SIZE);
+	pmd_populate_kernel(&init_mm, pmdp, ptep);
+
+	resume_remote_cpus();
+	preempt_enable();
+}
+
+#define vmemmap_flush_tlb_all vmemmap_flush_tlb_all
+static inline void vmemmap_flush_tlb_all(void)
+{
+}
+
+#define vmemmap_flush_tlb_range vmemmap_flush_tlb_range
+static inline void vmemmap_flush_tlb_range(unsigned long start, unsigned long end)
+{
+}
+
+#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */
+
 #endif
mm/hugetlb_vmemmap.c +14 -0
index 2dd92e58f304..893c73493d9c 100644
@@ -46,6 +46,18 @@ struct vmemmap_remap_walk {
 	unsigned long		flags;
 };
 
+#ifndef vmemmap_update_lock
+static void vmemmap_update_lock(void)
+{
+}
+#endif
+
+#ifndef vmemmap_update_unlock
+static void vmemmap_update_unlock(void)
+{
+}
+#endif
+
 #ifndef vmemmap_update_pmd
 static inline void vmemmap_update_pmd(unsigned long addr,
 				      pmd_t *pmdp, pte_t *ptep)
@@ -194,10 +206,12 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
 
 	VM_BUG_ON(!PAGE_ALIGNED(start | end));
 
+	vmemmap_update_lock();
 	mmap_read_lock(&init_mm);
 	ret = walk_page_range_novma(&init_mm, start, end, &vmemmap_remap_ops,
 				    NULL, walk);
 	mmap_read_unlock(&init_mm);
+	vmemmap_update_unlock();
 	if (ret)
 		return ret;
 
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
author
#7 lore raw

Re: [RFC PATCH 3/4] arm64: pause remote CPUs to update vmemmap

Quoted 47 lines · Yu Zhao

Pause remote CPUs so that the local CPU can follow the proper BBM sequence to safely update the vmemmap mapping struct page areas.

While updating the vmemmap, it is guaranteed that neither the local CPU nor the remote ones will access the struct page area being updated, and therefore they will not trigger kernel PFs.

---
 arch/arm64/include/asm/pgalloc.h | 55 ++++++++++++++++++++++++++++++++
 mm/hugetlb_vmemmap.c             | 14 ++++++++
 2 files changed, 69 insertions(+)
arch/arm64/include/asm/pgalloc.h +18 -0
index 8ff5f2a2579e..1af1aa34a351 100644
@@ -12,6 +12,7 @@
 #include <asm/processor.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
+#include <asm/cpu.h>
 
 #define __HAVE_ARCH_PGD_FREE
 #define __HAVE_ARCH_PUD_FREE
@@ -137,4 +138,58 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
        __pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE | PMD_TABLE_PXN);
 }
 
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+
+#define vmemmap_update_lock vmemmap_update_lock
+static inline void vmemmap_update_lock(void)
+{
+       cpus_read_lock();
+}
+
+#define vmemmap_update_unlock vmemmap_update_unlock
+static inline void vmemmap_update_unlock(void)
+{
+       cpus_read_unlock();
+}
+
+#define vmemmap_update_pte vmemmap_update_pte
+static inline void vmemmap_update_pte(unsigned long addr, pte_t *ptep, pte_t pte)
+{

arch/arm64/include/asm/pgalloc.h pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) · line 167

Quoted 10 lines
arch/arm64/include/asm/pgalloc.h +10 -0
+       preempt_disable();
+       pause_remote_cpus();
+
+       pte_clear(&init_mm, addr, ptep);
+       flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+       set_pte_at(&init_mm, addr, ptep, pte);
+
+       resume_remote_cpus();
+       preempt_enable();
+}

Note that I kept this API from Nanyong for the sake of discussion. What I actually plan to test in our production is:

#define vmemmap_update_pte_range_start vmemmap_update_pte_range_start static inline void vmemmap_update_pte_range_start(pte_t *pte,
unsigned long start,
unsigned long end)

{
        preempt_disable();
        pause_remote_cpus();

        for (; start != end; start += PAGE_SIZE, pte++)
                pte_clear(&init_mm, start, pte);

        flush_tlb_kernel_range(start, end);
}

#define vmemmap_update_pte_range_end vmemmap_update_pte_range_end static inline void vmemmap_update_pte_range_end(void)

{
        resume_remote_cpus();
        preempt_enable();
}
Quoted 66 lines
arch/arm64/include/asm/pgalloc.h +26 -0
+#define vmemmap_update_pmd vmemmap_update_pmd
+static inline void vmemmap_update_pmd(unsigned long addr, pmd_t *pmdp, pte_t *ptep)
+{
+       preempt_disable();
+       pause_remote_cpus();
+
+       pmd_clear(pmdp);
+       flush_tlb_kernel_range(addr, addr + PMD_SIZE);
+       pmd_populate_kernel(&init_mm, pmdp, ptep);
+
+       resume_remote_cpus();
+       preempt_enable();
+}
+
+#define vmemmap_flush_tlb_all vmemmap_flush_tlb_all
+static inline void vmemmap_flush_tlb_all(void)
+{
+}
+
+#define vmemmap_flush_tlb_range vmemmap_flush_tlb_range
+static inline void vmemmap_flush_tlb_range(unsigned long start, unsigned long end)
+{
+}
+
+#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */
+
 #endif
mm/hugetlb_vmemmap.c +14 -1
index 2dd92e58f304..893c73493d9c 100644
@@ -46,6 +46,18 @@ struct vmemmap_remap_walk {
        unsigned long           flags;
 };
 
+#ifndef vmemmap_update_lock
+static void vmemmap_update_lock(void)
+{
+}
+#endif
+
+#ifndef vmemmap_update_unlock
+static void vmemmap_update_unlock(void)
+{
+}
+#endif
+
 #ifndef vmemmap_update_pmd
 static inline void vmemmap_update_pmd(unsigned long addr,
                                      pmd_t *pmdp, pte_t *ptep)
@@ -194,10 +206,12 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
 
        VM_BUG_ON(!PAGE_ALIGNED(start | end));
 
+       vmemmap_update_lock();
        mmap_read_lock(&init_mm);
        ret = walk_page_range_novma(&init_mm, start, end, &vmemmap_remap_ops,
                                    NULL, walk);
        mmap_read_unlock(&init_mm);
+       vmemmap_update_unlock();
        if (ret)
                return ret;
 
--

2.46.0.rc2.264.g509ed76dc8-goog

author
#8 lore raw

RFC PATCH 4/4 arm64: mm: Re-enable OPTIMIZE_HUGETLB_VMEMMAP

In reply to Yu Zhao

From: Nanyong Sun <[email protected]>

Now update of vmemmap page table can follow the rule of break-before-make safely for arm64 architecture, re-enable HVO on arm64.

---
 arch/arm64/Kconfig | 1 +
 1 file changed, 1 insertion(+)
arch/arm64/Kconfig +1 -0
index a2f8ff354ca6..25ff026cdaf5 100644
@@ -105,6 +105,7 @@ config ARM64
 	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
 	select ARCH_WANT_FRAME_POINTERS
 	select ARCH_WANT_HUGE_PMD_SHARE if ARM64_4K_PAGES || (ARM64_16K_PAGES && !ARM64_VA_BITS_36)
+	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANTS_EXECMEM_LATE if EXECMEM
 	select ARCH_WANTS_NO_INSTR
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog