mm/arm64: re-enable HVO
Subsystems (3)
cover mm/arm64: re-enable HVO
This series presents one of the previously discussed approaches to re-enable HugeTLB Vmemmap Optimization (HVO) on arm64. HVO was disabled by commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the following reason:
This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.
Other approaches that have been discussed include:
A. Handle kernel PF while doing BBM [1],
B. Use stop_machine() while doing BBM [2], and,
C. Enable FEAT_BBM level 2 and keep the memory contents at the old
and new output addresses unchanged to avoid BBM (D8.16.1-2) [3].
A quick comparison between this approach (D) and the above approaches:
--+------------------------------+-----------------------------+
| Pro | Con |
--+------------------------------+-----------------------------+
A | Low latency, h/w independent | Predictability concerns [4] | B | Predictable, h/w independent | High latency | C | Predictable, low latency | H/w dependent, complex | D | Predictable, h/w independent | Medium latency |
--+------------------------------+-----------------------------+
[1] https://lore.kernel.org/[email protected]/
[2] https://lore.kernel.org/[email protected]/
[3] https://lore.kernel.org/[email protected]/
[4] https://lore.kernel.org/20240326125409.GA9552@willie-the-truck/
Nanyong Sun (2):
mm: HVO: introduce helper function to update and flush pgtable
arm64: mm: Re-enable OPTIMIZE_HUGETLB_VMEMMAP
Yu Zhao (2):
arm64: use IPIs to pause/resume remote CPUs
arm64: pause remote CPUs to update vmemmap
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/pgalloc.h | 55 ++++++++++++++++ arch/arm64/include/asm/smp.h | 3 + arch/arm64/kernel/smp.c | 110 +++++++++++++++++++++++++++++++ mm/hugetlb_vmemmap.c | 69 +++++++++++++++---- 5 files changed, 226 insertions(+), 12 deletions(-)
base-commit: de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
RFC PATCH 1/4 mm: HVO: introduce helper function to update and flush pgtable
From: Nanyong Sun <[email protected]>
Add pmd/pte update and tlb flush helper function to update page table. This refactoring patch is designed to facilitate each architecture to implement its own special logic in preparation for the arm64 architecture to follow the necessary break-before-make sequence when updating page tables.
--- mm/hugetlb_vmemmap.c | 55 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-)
mm/hugetlb_vmemmap.c +43 -12
@@ -46,6 +46,37 @@ struct vmemmap_remap_walk {unsigned long flags; }; +#ifndef vmemmap_update_pmd +static inline void vmemmap_update_pmd (unsigned long addr, +pmd_t *pmdp,pte_t *ptep) +{ +pmd_populate_kernel (&init_mm, pmdp, ptep); +} +#endif + +#ifndef vmemmap_update_pte +static inline void vmemmap_update_pte (unsigned long addr, +pte_t *ptep,pte_t pte) +{ +set_pte_at (&init_mm, addr, ptep, pte); +} +#endif + +#ifndef vmemmap_flush_tlb_all +static inline void vmemmap_flush_tlb_all (void ) +{ +flush_tlb_all (); +} +#endif + +#ifndef vmemmap_flush_tlb_range +static inline void vmemmap_flush_tlb_range (unsigned long start, +unsigned long end) +{ +flush_tlb_kernel_range (start, end); +} +#endif +static int vmemmap_split_pmd (pmd_t *pmd,struct page *head,unsigned long start,struct vmemmap_remap_walk *walk) {
@@ -81,9 +112,9 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,/* Make pte visible before pmd. See comment in pmd_install(). */ smp_wmb (); -pmd_populate_kernel (&init_mm, pmd, pgtable); +vmemmap_update_pmd (start, pmd, pgtable);if (!(walk->flags &VMEMMAP_SPLIT_NO_TLB_FLUSH )) -flush_tlb_kernel_range (start, start +PMD_SIZE ); +vmemmap_flush_tlb_range (start, start +PMD_SIZE ); }else {pte_free_kernel (&init_mm, pgtable); }
@@ -171,7 +202,7 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,return ret;if (walk->remap_pte && !(walk->flags &VMEMMAP_REMAP_NO_TLB_FLUSH )) -flush_tlb_kernel_range (start, end); +vmemmap_flush_tlb_range (start, end);return 0 ; }
@@ -220,15 +251,15 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,/* * Makes sure that preceding stores to the page contents from -* vmemmap_remap_free() become visible before the set_pte_at() -* write. +* vmemmap_remap_free() become visible before the +* vmemmap_update_pte() write. */ smp_wmb (); } entry =mk_pte (walk->reuse_page, pgprot);list_add (&page->lru, walk->vmemmap_pages); -set_pte_at (&init_mm, addr, pte, entry); +vmemmap_update_pte (addr, pte, entry); }/*
@@ -267,10 +298,10 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,/* * Makes sure that preceding stores to the page contents become visible -* before the set_pte_at() write. +* before the vmemmap_update_pte() write. */ smp_wmb (); -set_pte_at (&init_mm, addr, pte,mk_pte (page, pgprot)); +vmemmap_update_pte (addr, pte,mk_pte (page, pgprot)); }/**
@@ -536,7 +567,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, }if (restored) -flush_tlb_all (); +vmemmap_flush_tlb_all ();if (!ret) ret = restored;return ret;
@@ -664,7 +695,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_lbreak ; } -flush_tlb_all (); +vmemmap_flush_tlb_all ();/* avoid writes from page_ref_add_unless() while folding vmemmap */ synchronize_rcu ();
@@ -684,7 +715,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l* allowing more vmemmap remaps to occur. */ if (ret == -ENOMEM && !list_empty (&vmemmap_pages)) { -flush_tlb_all (); +vmemmap_flush_tlb_all ();free_vmemmap_page_list (&vmemmap_pages);INIT_LIST_HEAD (&vmemmap_pages); __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages,
@@ -692,7 +723,7 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l } } -flush_tlb_all (); +vmemmap_flush_tlb_all ();free_vmemmap_page_list (&vmemmap_pages); }
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
RFC PATCH 2/4 arm64: use IPIs to pause/resume remote CPUs
In reply to Yu ZhaoUse pseudo-NMI IPIs to pause remote CPUs for a short period of time, and then reliably resume them when the local CPU exits critical sections that preclude the execution of remote CPUs.
A typical example of such critical sections is BBM on kernel PTEs. HugeTLB Vmemmap Optimization (HVO) on arm64 was disabled by commit 060a2c92d1b6 ("arm64: mm: hugetlb: Disable HUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the folllowing reason:
This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.
Supporting BBM on kernel PTEs is one of the approaches that can potentially make arm64 support HVO.
--- arch/arm64/include/asm/smp.h | 3 + arch/arm64/kernel/smp.c | 110 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+)
arch/arm64/include/asm/smp.h +3 -0
@@ -133,6 +133,9 @@ bool cpus_are_stuck_in_kernel(void);extern void crash_smp_send_stop (void );extern bool smp_crash_stop_failed (void ); +void pause_remote_cpus (void ); +void resume_remote_cpus (void ); +#endif /* ifndef __ASSEMBLY__ */ #endif /* ifndef __ASM_SMP_H */
arch/arm64/kernel/smp.c +110 -0
173 lines not on this page · raw message
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
Re: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs
[...]
Quoted 12 lines
arch/arm64/kernel/smp.c +11
-0+ +void resume_remote_cpus (void ) +{ +cpumask_t cpus_to_resume; + +lockdep_assert_cpus_held (); +lockdep_assert_preemption_disabled (); + +cpumask_copy (&cpus_to_resume, cpu_online_mask); +cpumask_clear_cpu (smp_processor_id (), &cpus_to_resume); +
arch/arm64/kernel/smp.c void __noreturn panic_smp_self_stop(void) · line 979
Quoted 10 lines
arch/arm64/kernel/smp.c +10
-0+spin_lock (&cpu_pause_lock); + +cpumask_setall (&resumed_cpus); +/* A typical example for sleep and wake-up functions. */ +smp_mb (); +while (cpumask_intersects (&cpus_to_resume, &paused_cpus)) { +sev (); +cpu_relax (); +barrier (); + }
I'm curious, is there a fundamental reason why we wait for paused CPUs to actually start running, or is it simply easier to get the implementation race-free, in particular when we have two pause_remote_cpus() calls shortly after each other and another remote CPU might still be on its way out of pause_local_cpu() from the first pause.
Signature · 3 lines
Cheers, David / dhildenb
kernel test robot
build botRe: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs
In reply to Yu Zhaokernel test robot noticed the following build warnings:
Hi Yu,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build warnings:
[auto build test WARNING on de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed]
url: https://github.com/intel-lab-lkp/linux/commits/Yu-Zhao/mm-HVO-introduce-helper-function-to-update-and-flush-pgtable/20240806-104445 base: de9c2c66ad8e787abec7c9d7eff4f8c3cdd28aed patch link: https://lore.kernel.org/r/20240806022114.3320543-3-yuzhao%40google.com patch subject: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs config: arm64-allnoconfig (https://download.01.org/0day-ci/archive/20240808/[email protected]/config) compiler: aarch64-linux-gcc (GCC) 14.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240808/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> arch/arm64/kernel/smp.c:988:24: warning: 'ipi_cpu_crash_stop' defined but not used [-Wunused-function]
988 | static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
| ^~~~~~~~~~~~~~~~~~
vim +/ipi_cpu_crash_stop +988 arch/arm64/kernel/smp.c
78fd584cdec0518 AKASHI Takahiro 2017-04-03 987
5ab6876c7843db5 Josh Poimboeuf 2023-04-12 @988 static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
78fd584cdec0518 AKASHI Takahiro 2017-04-03 989 {
78fd584cdec0518 AKASHI Takahiro 2017-04-03 990 #ifdef CONFIG_KEXEC_CORE
78fd584cdec0518 AKASHI Takahiro 2017-04-03 991 crash_save_cpu(regs, cpu);
78fd584cdec0518 AKASHI Takahiro 2017-04-03 992
78fd584cdec0518 AKASHI Takahiro 2017-04-03 993 atomic_dec(&waiting_for_crash_ipi);
78fd584cdec0518 AKASHI Takahiro 2017-04-03 994
78fd584cdec0518 AKASHI Takahiro 2017-04-03 995 local_irq_disable();
f5df26961853d68 James Morse 2018-01-08 996 sdei_mask_local_cpu();
78fd584cdec0518 AKASHI Takahiro 2017-04-03 997
de58ed5e16e62f3 Gavin Shan 2020-03-19 998 if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
de58ed5e16e62f3 Gavin Shan 2020-03-19 999 __cpu_try_die(cpu);
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1000
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1001 /* just in case */
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1002 cpu_park_loop();
5ab6876c7843db5 Josh Poimboeuf 2023-04-12 1003 #else
5ab6876c7843db5 Josh Poimboeuf 2023-04-12 1004 BUG();
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1005 #endif
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1006 }
78fd584cdec0518 AKASHI Takahiro 2017-04-03 1007 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [RFC PATCH 2/4] arm64: use IPIs to pause/resume remote CPUs
In reply to Yu ZhaoHi,
Quoted 15 lines · Yu Zhao
Use pseudo-NMI IPIs to pause remote CPUs for a short period of time, and then reliably resume them when the local CPU exits critical sections that preclude the execution of remote CPUs.
A typical example of such critical sections is BBM on kernel PTEs. HugeTLB Vmemmap Optimization (HVO) on arm64 was disabled by commit
060a2c92d1b6("arm64: mm: hugetlb: DisableHUGETLB_PAGE_OPTIMIZE_VMEMMAP") due to the folllowing reason:This is deemed UNPREDICTABLE by the Arm architecture without a break-before-make sequence (make the PTE invalid, TLBI, write the new valid PTE). However, such sequence is not possible since the vmemmap may be concurrently accessed by the kernel.
Quoted 8 lines
Supporting BBM on kernel PTEs is one of the approaches that can potentially make arm64 support HVO.
Signed-off-by Yu Zhao <[email protected]>--- arch/arm64/include/asm/smp.h | 3 + arch/arm64/kernel/smp.c | 110 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+)
I'm a bit curious how your approach is reliable / performant in all cases. As far as I understand it:
1. Patch #4 in your series unconditionally turns on "ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP" for arm64.
2. In order for it to work reliably, you need the "pause all CPUs" functionality introduced in this patch.
3. In order for the "pause all CPUs" functionality to be performant you need NMI or, at least, pseudo-NMI to be used to pause all CPUs.
4. Even when you configure the kernel for pseudo-NMI it's not 100% guaranteed that pseudo-NMI will be turned on. Specifically:
4a) There's an extra kernel command line parameter you need to actually enable pseudo-NMI. We can debate about the inability to turn on pseudo-NMI without the command line parameter, but at the moment it's there because pseudo-NMI has some performance implications. Apparently these performance implications are more non-trivial on some early arm64 CPUs.
4b) Even if we changed it so that the command-line parameter wasn't needed, there are still some boards out there that are known not to be able to enable pseudo-NMI. There are certainly some Mediatek Chromebooks that have a BIOS bug making pseudo-NMI unreliable. See the mediatek,broken-save-restore-fw device tree property. ...and even if you ignore the Mediatek Chromebooks, there's at least one more system I know of that's broken with pseudo-NMI. Since you're at Google, you could look at b/308278090 for details but the quick summary is that some devices running a TEE are hanging when pseudo NMI is enabled. ...and, even if that's fixed, it feels somewhat likely that there are other systems where pseudo-NMI won't be usable.
Unless I'm misunderstanding, it feels like anything you have that relies on NMI/pseudo-NMI needs to fall back safely/reliably if NMI/pseudo-NMI isn't there.
Quoted 22 lines
arch/arm64/include/asm/smp.h +3
-0@@ -133,6 +133,9 @@ bool cpus_are_stuck_in_kernel(void);extern void crash_smp_send_stop (void );extern bool smp_crash_stop_failed (void ); +void pause_remote_cpus (void ); +void resume_remote_cpus (void ); +#endif /* ifndef __ASSEMBLY__ */ #endif /* ifndef __ASM_SMP_H */ arch/arm64/kernel/smp.c +0
-0@@ -68,16 +68,25 @@ enum ipi_msg_type {IPI_RESCHEDULE ,IPI_CALL_FUNC , IPI_CPU_STOP,
arch/arm64/kernel/smp.c enum ipi_msg_type { · line 80
Quoted 10 lines
arch/arm64/kernel/smp.c +7
-0+ IPI_CPU_PAUSE, +#ifdef CONFIG_KEXEC_CORE IPI_CPU_CRASH_STOP, +#endif +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST IPI_TIMER, +#endif +#ifdef CONFIG_IRQ_WORK IPI_IRQ_WORK, +#endif
I assume all these "ifdefs" are there because this adds up to more than 8 IPIs. That means that someone wouldn't be able to enable all of these things, right? Feels like we'd want to solve this before landing things. In the least it would be good if this built upon:
https://lore.kernel.org/r/20240625160718.v2.1.Id4817adef610302554b8aa42b090d57270dc119c@changeid/
...and then maybe we could figure out if there are other ways to consolidate NMIs. Previously, for instance, we had the "KGDB" and "backtrace" IPIs combined into one but we split them upon review feedback. If necessary they would probably be easy to re-combine.
RFC PATCH 3/4 arm64: pause remote CPUs to update vmemmap
In reply to Yu ZhaoPause remote CPUs so that the local CPU can follow the proper BBM sequence to safely update the vmemmap mapping struct page areas.
While updating the vmemmap, it is guaranteed that neither the local CPU nor the remote ones will access the struct page area being updated, and therefore they will not trigger kernel PFs.
--- arch/arm64/include/asm/pgalloc.h | 55 ++++++++++++++++++++++++++++++++ mm/hugetlb_vmemmap.c | 14 ++++++++ 2 files changed, 69 insertions(+)
arch/arm64/include/asm/pgalloc.h +55 -0
@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> +#include <asm/cpu.h> #define __HAVE_ARCH_PGD_FREE#define __HAVE_ARCH_PUD_FREE
@@ -137,4 +138,58 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)__pmd_populate (pmdp,page_to_phys (ptep),PMD_TYPE_TABLE |PMD_TABLE_PXN ); } +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + +#define vmemmap_update_lock vmemmap_update_lock +static inline void vmemmap_update_lock (void ) +{ +cpus_read_lock (); +} + +#define vmemmap_update_unlock vmemmap_update_unlock +static inline void vmemmap_update_unlock (void ) +{ +cpus_read_unlock (); +} + +#define vmemmap_update_pte vmemmap_update_pte +static inline void vmemmap_update_pte (unsigned long addr,pte_t *ptep,pte_t pte) +{ +preempt_disable (); +pause_remote_cpus (); + +pte_clear (&init_mm, addr, ptep); +flush_tlb_kernel_range (addr, addr +PAGE_SIZE ); +set_pte_at (&init_mm, addr, ptep, pte); + +resume_remote_cpus (); +preempt_enable (); +} + +#define vmemmap_update_pmd vmemmap_update_pmd +static inline void vmemmap_update_pmd (unsigned long addr,pmd_t *pmdp,pte_t *ptep) +{ +preempt_disable (); +pause_remote_cpus (); + +pmd_clear (pmdp); +flush_tlb_kernel_range (addr, addr +PMD_SIZE ); +pmd_populate_kernel (&init_mm, pmdp, ptep); + +resume_remote_cpus (); +preempt_enable (); +} + +#define vmemmap_flush_tlb_all vmemmap_flush_tlb_all +static inline void vmemmap_flush_tlb_all (void ) +{ +} + +#define vmemmap_flush_tlb_range vmemmap_flush_tlb_range +static inline void vmemmap_flush_tlb_range (unsigned long start,unsigned long end) +{ +} + +#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */ +#endif
mm/hugetlb_vmemmap.c +14 -0
@@ -46,6 +46,18 @@ struct vmemmap_remap_walk {unsigned long flags; }; +#ifndef vmemmap_update_lock +static void vmemmap_update_lock (void ) +{ +} +#endif + +#ifndef vmemmap_update_unlock +static void vmemmap_update_unlock (void ) +{ +} +#endif +#ifndef vmemmap_update_pmdstatic inline void vmemmap_update_pmd (unsigned long addr,pmd_t *pmdp,pte_t *ptep)
@@ -194,10 +206,12 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,VM_BUG_ON (!PAGE_ALIGNED (start | end)); +vmemmap_update_lock ();mmap_read_lock (&init_mm); ret =walk_page_range_novma (&init_mm, start, end, &vmemmap_remap_ops,NULL , walk);mmap_read_unlock (&init_mm); +vmemmap_update_unlock ();if (ret)return ret;
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog
Re: [RFC PATCH 3/4] arm64: pause remote CPUs to update vmemmap
Quoted 47 lines · Yu Zhao
Pause remote CPUs so that the local CPU can follow the proper BBM sequence to safely update the vmemmap mapping
struct pageareas.While updating the vmemmap, it is guaranteed that neither the local CPU nor the remote ones will access the
struct pagearea being updated, and therefore they will not trigger kernel PFs.Signed-off-by Yu Zhao <[email protected]>--- arch/arm64/include/asm/pgalloc.h | 55 ++++++++++++++++++++++++++++++++ mm/hugetlb_vmemmap.c | 14 ++++++++ 2 files changed, 69 insertions(+)arch/arm64/include/asm/pgalloc.h +18
-0@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> +#include <asm/cpu.h> #define __HAVE_ARCH_PGD_FREE#define __HAVE_ARCH_PUD_FREE@@ -137,4 +138,58 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)__pmd_populate (pmdp,page_to_phys (ptep),PMD_TYPE_TABLE |PMD_TABLE_PXN ); } +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + +#define vmemmap_update_lock vmemmap_update_lock +static inline void vmemmap_update_lock (void ) +{ +cpus_read_lock (); +} + +#define vmemmap_update_unlock vmemmap_update_unlock +static inline void vmemmap_update_unlock (void ) +{ +cpus_read_unlock (); +} + +#define vmemmap_update_pte vmemmap_update_pte +static inline void vmemmap_update_pte (unsigned long addr,pte_t *ptep,pte_t pte) +{
arch/arm64/include/asm/pgalloc.h pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) · line 167
Quoted 10 lines
arch/arm64/include/asm/pgalloc.h +10
-0+preempt_disable (); +pause_remote_cpus (); + +pte_clear (&init_mm, addr, ptep); +flush_tlb_kernel_range (addr, addr +PAGE_SIZE ); +set_pte_at (&init_mm, addr, ptep, pte); + +resume_remote_cpus (); +preempt_enable (); +}
Note that I kept this API from Nanyong for the sake of discussion. What I actually plan to test in our production is:
#define vmemmap_update_pte_range_start vmemmap_update_pte_range_start static inline void vmemmap_update_pte_range_start(pte_t *pte,
unsigned long start,
unsigned long end)
{
preempt_disable();
pause_remote_cpus();
for (; start != end; start += PAGE_SIZE, pte++)
pte_clear(&init_mm, start, pte);
flush_tlb_kernel_range(start, end);
}#define vmemmap_update_pte_range_end vmemmap_update_pte_range_end static inline void vmemmap_update_pte_range_end(void)
{
resume_remote_cpus();
preempt_enable();
}Quoted 66 lines
arch/arm64/include/asm/pgalloc.h +26
-0+#define vmemmap_update_pmd vmemmap_update_pmd +static inline void vmemmap_update_pmd (unsigned long addr,pmd_t *pmdp,pte_t *ptep) +{ +preempt_disable (); +pause_remote_cpus (); + +pmd_clear (pmdp); +flush_tlb_kernel_range (addr, addr +PMD_SIZE ); +pmd_populate_kernel (&init_mm, pmdp, ptep); + +resume_remote_cpus (); +preempt_enable (); +} + +#define vmemmap_flush_tlb_all vmemmap_flush_tlb_all +static inline void vmemmap_flush_tlb_all (void ) +{ +} + +#define vmemmap_flush_tlb_range vmemmap_flush_tlb_range +static inline void vmemmap_flush_tlb_range (unsigned long start,unsigned long end) +{ +} + +#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */ +#endif mm/hugetlb_vmemmap.c +14
-1@@ -46,6 +46,18 @@ struct vmemmap_remap_walk {unsigned long flags; }; +#ifndef vmemmap_update_lock +static void vmemmap_update_lock (void ) +{ +} +#endif + +#ifndef vmemmap_update_unlock +static void vmemmap_update_unlock (void ) +{ +} +#endif +#ifndef vmemmap_update_pmdstatic inline void vmemmap_update_pmd (unsigned long addr,pmd_t *pmdp,pte_t *ptep)@@ -194,10 +206,12 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,VM_BUG_ON (!PAGE_ALIGNED (start | end)); +vmemmap_update_lock ();mmap_read_lock (&init_mm); ret =walk_page_range_novma (&init_mm, start, end, &vmemmap_remap_ops,NULL , walk);mmap_read_unlock (&init_mm); +vmemmap_update_unlock ();if (ret)return ret; --2.46.0.rc2.264.g509ed76dc8-goog
RFC PATCH 4/4 arm64: mm: Re-enable OPTIMIZE_HUGETLB_VMEMMAP
In reply to Yu ZhaoFrom: Nanyong Sun <[email protected]>
Now update of vmemmap page table can follow the rule of break-before-make safely for arm64 architecture, re-enable HVO on arm64.
--- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/arm64/Kconfig +1 -0
@@ -105,6 +105,7 @@ config ARM64 select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_HUGE_PMD_SHARE if ARM64_4K_PAGES || (ARM64_16K_PAGES && !ARM64_VA_BITS_36) + select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANTS_EXECMEM_LATE if EXECMEM select ARCH_WANTS_NO_INSTR
Signature · 1 line
2.46.0.rc2.264.g509ed76dc8-goog