hugetlb: arm64: add mte support
Subsystems (5)
PATCH v3 1/2 hugetlb: arm64: add mte support
Enable MTE support for hugetlb.
The MTE page flags will be set on the folio only. When copying hugetlb folio (for example, CoW), the tags for all subpages will be copied when copying the first subpage.
When freeing hugetlb folio, the MTE flags will be cleared.
--- arch/arm64/include/asm/hugetlb.h | 15 ++++++- arch/arm64/include/asm/mman.h | 3 +- arch/arm64/include/asm/mte.h | 67 ++++++++++++++++++++++++++++++++ arch/arm64/kernel/hibernate.c | 7 ++++ arch/arm64/kernel/mte.c | 25 +++++++++++- arch/arm64/kvm/guest.c | 16 ++++++-- arch/arm64/kvm/mmu.c | 11 ++++++ arch/arm64/mm/copypage.c | 33 +++++++++++++--- fs/hugetlbfs/inode.c | 2 +- 9 files changed, 166 insertions(+), 13 deletions(-)
v3: * Fixed the build error when !CONFIG_ARM64_MTE.
* Incorporated the comment from David to have hugetlb folio specific APIs for manipulating the page flags.
* Don't assume the first page is the head page since huge page copy can start from any subpage.
v2: * Reimplemented the patch to fix the comments from Catalin.
* Added test cases (patch #2) per Catalin.
arch/arm64/include/asm/hugetlb.h +14 -1
@@ -11,6 +11,7 @@#define __ASM_HUGETLB_H#include <asm/cacheflush.h> +#include <asm/mte.h> #include <asm/page.h> #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
@@ -18,9 +19,21 @@extern bool arch_hugetlb_migration_supported (struct hstate *h);#endif +#ifdef CONFIG_ARM64_MTE +#define CLEAR_FLAGS (BIT(PG_dcache_clean) | BIT(PG_mte_tagged) | \ + BIT(PG_mte_lock)) +#else +#define CLEAR_FLAGS BIT(PG_dcache_clean) +#endif +static inline void arch_clear_hugetlb_flags (struct folio *folio) { -clear_bit (PG_dcache_clean, &folio->flags); +if (!system_supports_mte ()) { +clear_bit (PG_dcache_clean, &folio->flags); +return ; + } + + folio->flags &= ~CLEAR_FLAGS ; }#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
arch/arm64/include/asm/mman.h +2 -1
@@ -28,7 +28,8 @@ static inline unsigned long arch_calc_vm_flag_bits(unsigned long flags)* backed by tags-capable memory. The vm_flags may be overridden by a * filesystem supporting MTE (RAM-based). */ -if (system_supports_mte () && (flags &MAP_ANONYMOUS )) +if (system_supports_mte () && + (flags & (MAP_ANONYMOUS |MAP_HUGETLB )))return VM_MTE_ALLOWED ;return 0 ;
arch/arm64/include/asm/mte.h +67 -0
91 lines not on this page · raw message
arch/arm64/kernel/hibernate.c +7 -0
@@ -266,10 +266,17 @@ static int swsusp_mte_save_tags(void) max_zone_pfn =zone_end_pfn (zone);for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {struct page *page =pfn_to_online_page (pfn); +struct folio *folio;if (!page)continue ; + folio =page_folio (page); + +if (folio_test_hugetlb (folio) && + !folio_hugetlb_mte_tagged (folio)) +continue ; +if (!page_mte_tagged (page))continue ;
arch/arm64/kernel/mte.c +23 -2
@@ -38,7 +38,22 @@ EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);void mte_sync_tags (pte_t pte,unsigned int nr_pages) {struct page *page =pte_page (pte); -unsigned int i; +struct folio *folio =page_folio (page); +unsigned long i; + +if (folio_test_hugetlb (folio)) { +unsigned long nr =folio_nr_pages (folio); +/* Hugetlb MTE flags are set for head page only */ +if (try_folio_hugetlb_mte_tagging (folio)) { +for (i =0 ; i < nr; i++, page++) +mte_clear_page_tags (page_address (page)); +set_folio_hugetlb_mte_tagged (folio); + } + +smp_wmb (); + +return ; + }/* if PG_mte_tagged is set, tags have already been initialised */ for (i =0 ; i < nr_pages; i++, page++) {
@@ -410,6 +425,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,void *maddr;struct page *page =get_user_page_vma_remote (mm, addr, gup_flags, &vma); +struct folio *folio;if (IS_ERR (page)) { err =PTR_ERR (page);
@@ -428,7 +444,12 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,put_page (page);break ; } -WARN_ON_ONCE (!page_mte_tagged (page)); + + folio =page_folio (page); +if (folio_test_hugetlb (folio)) +WARN_ON_ONCE (!folio_hugetlb_mte_tagged (folio)); +else +WARN_ON_ONCE (!page_mte_tagged (page));/* limit access to the end of the page */ offset =offset_in_page (addr);
arch/arm64/kvm/guest.c +13 -3
@@ -1050,6 +1050,7 @@ int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,void *maddr;unsigned long num_tags;struct page *page; +struct folio *folio;if (is_error_noslot_pfn (pfn)) { ret = -EFAULT ;
@@ -1062,10 +1063,13 @@ int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, ret = -EFAULT ; goto out; } + folio =page_folio (page); maddr =page_address (page);if (!write) { -if (page_mte_tagged (page)) +if ((folio_test_hugetlb (folio) && +folio_hugetlb_mte_tagged (folio)) || +page_mte_tagged (page)) num_tags =mte_copy_tags_to_user (tags, maddr,MTE_GRANULES_PER_PAGE );else
@@ -1079,14 +1083,20 @@ int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,* __set_ptes() in the VMM but still overriding the * tags, hence ignoring the return value. */ -try_page_mte_tagging (page); +if (folio_test_hugetlb (folio)) +try_folio_hugetlb_mte_tagging (folio); +else +try_page_mte_tagging (page); num_tags =mte_copy_tags_from_user (maddr, tags,MTE_GRANULES_PER_PAGE );/* uaccess failed, don't leave stale tags */ if (num_tags !=MTE_GRANULES_PER_PAGE )mte_clear_page_tags (maddr); -set_page_mte_tagged (page); +if (folio_test_hugetlb (folio)) +set_folio_hugetlb_mte_tagged (folio); +else +set_page_mte_tagged (page);kvm_release_pfn_dirty (pfn); }
arch/arm64/kvm/mmu.c +11 -0
@@ -1401,10 +1401,21 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, {unsigned long i, nr_pages = size >>PAGE_SHIFT ;struct page *page =pfn_to_page (pfn); +struct folio *folio =page_folio (page);if (!kvm_has_mte (kvm))return ; +if (folio_test_hugetlb (folio)) { +/* Hugetlb has MTE flags set on head page only */ +if (try_folio_hugetlb_mte_tagging (folio)) { +for (i =0 ; i < nr_pages; i++, page++) +mte_clear_page_tags (page_address (page)); +set_folio_hugetlb_mte_tagged (folio); + } +return ; + } +for (i =0 ; i < nr_pages; i++, page++) {if (try_page_mte_tagging (page)) {mte_clear_page_tags (page_address (page));
arch/arm64/mm/copypage.c +28 -5
@@ -18,17 +18,40 @@ void copy_highpage(struct page *to, struct page *from) {void *kto =page_address (to);void *kfrom =page_address (from); +struct folio *src =page_folio (from); +struct folio *dst =page_folio (to); +unsigned int i, nr_pages;copy_page (kto, kfrom);if (kasan_hw_tags_enabled ())page_kasan_tag_reset (to); -if (system_supports_mte () &&page_mte_tagged (from)) { -/* It's a new page, shouldn't have been tagged yet */ -WARN_ON_ONCE (!try_page_mte_tagging (to)); -mte_copy_page_tags (kto, kfrom); -set_page_mte_tagged (to); +if (system_supports_mte ()) { +if (folio_test_hugetlb (src) &&folio_hugetlb_mte_tagged (src)) { +if (!try_folio_hugetlb_mte_tagging (dst)) +return ; + +/* +* Populate tags for all subpages. +* +* Don't assume the first page is head page since +* huge page copy may start from any subpage. +*/ + nr_pages =folio_nr_pages (src); +for (i =0 ; i < nr_pages; i++) { + kfrom =page_address (folio_page (src, i)); + kto =page_address (folio_page (dst, i)); +mte_copy_page_tags (kto, kfrom); + } +set_folio_hugetlb_mte_tagged (dst); + }else if (page_mte_tagged (from)) { +/* It's a new page, shouldn't have been tagged yet */ +WARN_ON_ONCE (!try_page_mte_tagging (to)); + +mte_copy_page_tags (kto, kfrom); +set_page_mte_tagged (to); + } } }EXPORT_SYMBOL (copy_highpage);
fs/hugetlbfs/inode.c +1 -1
@@ -110,7 +110,7 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)* way when do_mmap unwinds (may be important on powerpc * and ia64). */ -vm_flags_set (vma,VM_HUGETLB |VM_DONTEXPAND ); +vm_flags_set (vma,VM_HUGETLB |VM_DONTEXPAND |VM_MTE_ALLOWED ); vma->vm_ops = &hugetlb_vm_ops; ret =seal_check_write (info->seals, vma);
Signature · 1 line
2.41.0
PATCH v3 2/2 selftests: arm64: add hugetlb mte tests
The tests cover mmap, mprotect hugetlb with MTE prot and COW.
--- .../arm64/mte/check_hugetlb_options.c | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 tools/testing/selftests/arm64/mte/check_hugetlb_options.c
tools/testing/selftests/arm64/mte/check_hugetlb_options.c new file +285 -0
@@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2024 Ampere Computing LLC + +#define _GNU_SOURCE + +#include <errno.h> +#include <fcntl.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <ucontext.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include "kselftest.h" +#include "mte_common_util.h" +#include "mte_def.h" + +#define TAG_CHECK_ON 0 +#define TAG_CHECK_OFF 1 + +static unsigned long default_huge_page_size (void ) +{ +unsigned long hps =0 ; +char *line =NULL ; +size_t linelen =0 ; +FILE *f =fopen ("/proc/meminfo" ,"r" ); + +if (!f) +return 0 ; +while (getline (&line, &linelen, f) >0 ) { +if (sscanf (line,"Hugepagesize: %lu kB" , &hps) ==1 ) { + hps <<=10 ; +break ; + } + } + +free (line); +fclose (f); +return hps; +} + +static bool is_hugetlb_allocated (void ) +{ +unsigned long hps =0 ; +char *line =NULL ; +size_t linelen =0 ; +FILE *f =fopen ("/proc/meminfo" ,"r" ); + +if (!f) +return false; +while (getline (&line, &linelen, f) >0 ) { +if (sscanf (line,"Hugetlb: %lu kB" , &hps) ==1 ) { + hps <<=10 ; +break ; + } + } + +free (line); +fclose (f); + +if (hps >0 ) +return true; + +return false; +} + +static void write_sysfs (char *str,unsigned long val) +{ +FILE *f; + + f =fopen (str,"w" ); +if (!f) { +ksft_print_msg ("ERR: missing %s\n" , str); +return ; + } +fprintf (f,"%lu" , val); +fclose (f); +} + +static void allocate_hugetlb () +{ +write_sysfs ("/proc/sys/vm/nr_hugepages" ,2 ); +} + +static void free_hugetlb () +{ +write_sysfs ("/proc/sys/vm/nr_hugepages" ,0 ); +} + +static int check_child_tag_inheritance (char *ptr,int size,int mode) +{ +int i, parent_tag, child_tag, fault, child_status; +pid_t child; + + parent_tag =MT_FETCH_TAG ((uintptr_t )ptr); + fault =0 ; + + child =fork (); +if (child ==-1 ) { +ksft_print_msg ("FAIL: child process creation\n" ); +return KSFT_FAIL ; + }else if (child ==0 ) { +mte_initialize_current_context (mode, (uintptr_t )ptr, size); +/* Do copy on write */ +memset (ptr,'1' , size); +mte_wait_after_trig (); +if (cur_mte_cxt.fault_valid == true) { + fault =1 ; + goto check_child_tag_inheritance_err; + } +for (i =0 ; i < size; i +=MT_GRANULE_SIZE ) { + child_tag =MT_FETCH_TAG ((uintptr_t )(mte_get_tag_address (ptr + i))); +if (parent_tag != child_tag) { +ksft_print_msg ("FAIL: child mte tag (%d) mismatch\n" , i); + fault =1 ; + goto check_child_tag_inheritance_err; + } + } +check_child_tag_inheritance_err: +_exit (fault); + } +/* Wait for child process to terminate */ +wait (&child_status); +if (WIFEXITED (child_status)) + fault =WEXITSTATUS (child_status); +else + fault =1 ; +return (fault) ?KSFT_FAIL :KSFT_PASS ; +} + +static int check_mte_memory (char *ptr,int size,int mode,int tag_check) +{ +mte_initialize_current_context (mode, (uintptr_t )ptr, size); +memset (ptr,'1' , size); +mte_wait_after_trig (); +if (cur_mte_cxt.fault_valid == true) +return KSFT_FAIL ; + +return KSFT_PASS ; +} + +static int check_hugetlb_memory_mapping (int mem_type,int mode,int mapping,int tag_check) +{ +char *ptr, *map_ptr; +int result; +unsigned long map_size; + + map_size =default_huge_page_size (); + +mte_switch_mode (mode,MTE_ALLOW_NON_ZERO_TAG ); + map_ptr = (char *)mte_allocate_memory (map_size, mem_type, mapping, false); +if (check_allocated_memory (map_ptr, map_size, mem_type, false) !=KSFT_PASS ) +return KSFT_FAIL ; + +mte_initialize_current_context (mode, (uintptr_t )map_ptr, map_size); +/* Only mte enabled memory will allow tag insertion */ + ptr =mte_insert_tags ((void *)map_ptr, map_size); +if (!ptr || cur_mte_cxt.fault_valid == true) { +ksft_print_msg ("FAIL: Insert tags on anonymous mmap memory\n" ); +munmap ((void *)map_ptr, map_size); +return KSFT_FAIL ; + } + result =check_mte_memory (ptr, map_size, mode, tag_check); +mte_clear_tags ((void *)ptr, map_size); +mte_free_memory ((void *)map_ptr, map_size, mem_type, false); +if (result ==KSFT_FAIL ) +return KSFT_FAIL ; + +return KSFT_PASS ; +} + +static int check_clear_prot_mte_flag (int mem_type,int mode,int mapping) +{ +char *map_ptr; +int prot_flag, result; +unsigned long map_size; + + prot_flag =PROT_READ |PROT_WRITE ; +mte_switch_mode (mode,MTE_ALLOW_NON_ZERO_TAG ); + map_size =default_huge_page_size (); + map_ptr = (char *)mte_allocate_memory_tag_range (map_size, mem_type, mapping, +0 ,0 ); +if (check_allocated_memory_range (map_ptr, map_size, mem_type, +0 ,0 ) !=KSFT_PASS ) +return KSFT_FAIL ; +/* Try to clear PROT_MTE property and verify it by tag checking */ +if (mprotect (map_ptr, map_size, prot_flag)) { +mte_free_memory_tag_range ((void *)map_ptr, map_size, mem_type, +0 ,0 ); +ksft_print_msg ("FAIL: mprotect not ignoring clear PROT_MTE property\n" ); +return KSFT_FAIL ; + } + result =check_mte_memory (map_ptr, map_size, mode,TAG_CHECK_ON ); +mte_free_memory_tag_range ((void *)map_ptr, map_size, mem_type,0 ,0 ); +if (result !=KSFT_PASS ) +return KSFT_FAIL ; + +return KSFT_PASS ; +} + +static int check_child_hugetlb_memory_mapping (int mem_type,int mode,int mapping) +{ +char *ptr; +int result; +unsigned long map_size; + + map_size =default_huge_page_size (); + +mte_switch_mode (mode,MTE_ALLOW_NON_ZERO_TAG ); + ptr = (char *)mte_allocate_memory_tag_range (map_size, mem_type, mapping, +0 ,0 ); +if (check_allocated_memory_range (ptr, map_size, mem_type, +0 ,0 ) !=KSFT_PASS ) +return KSFT_FAIL ; + result =check_child_tag_inheritance (ptr, map_size, mode); +mte_free_memory_tag_range ((void *)ptr, map_size, mem_type,0 ,0 ); +if (result ==KSFT_FAIL ) +return result; + +return KSFT_PASS ; +} + +int main (int argc,char *argv[]) +{ +int err; + + err =mte_default_setup (); +if (err) +return err; + +/* Register signal handlers */ +mte_register_signal (SIGBUS , mte_default_handler); +mte_register_signal (SIGSEGV , mte_default_handler); + +allocate_hugetlb (); + +if (!is_hugetlb_allocated ()) { +ksft_print_msg ("ERR: Unable allocate hugetlb pages\n" ); +return KSFT_FAIL ; + } + +/* Set test plan */ +ksft_set_plan (12 ); + +mte_enable_pstate_tco (); + +evaluate_test (check_hugetlb_memory_mapping (USE_MMAP ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_OFF ), +"Check hugetlb memory with private mapping, sync error mode, mmap memory and tag check off\n" ); + +mte_disable_pstate_tco (); +evaluate_test (check_hugetlb_memory_mapping (USE_MMAP ,MTE_NONE_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_OFF ), +"Check hugetlb memory with private mapping, no error mode, mmap memory and tag check off\n" ); + +evaluate_test (check_hugetlb_memory_mapping (USE_MMAP ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_ON ), +"Check hugetlb memory with private mapping, sync error mode, mmap memory and tag check on\n" ); +evaluate_test (check_hugetlb_memory_mapping (USE_MPROTECT ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_ON ), +"Check hugetlb memory with private mapping, sync error mode, mmap/mprotect memory and tag check on\n" ); +evaluate_test (check_hugetlb_memory_mapping (USE_MMAP ,MTE_ASYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_ON ), +"Check hugetlb memory with private mapping, async error mode, mmap memory and tag check on\n" ); +evaluate_test (check_hugetlb_memory_mapping (USE_MPROTECT ,MTE_ASYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ,TAG_CHECK_ON ), +"Check hugetlb memory with private mapping, async error mode, mmap/mprotect memory and tag check on\n" ); + +evaluate_test (check_clear_prot_mte_flag (USE_MMAP ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check clear PROT_MTE flags with private mapping, sync error mode and mmap memory\n" ); +evaluate_test (check_clear_prot_mte_flag (USE_MPROTECT ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check clear PROT_MTE flags with private mapping and sync error mode and mmap/mprotect memory\n" ); + +evaluate_test (check_child_hugetlb_memory_mapping (USE_MMAP ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check child hugetlb memory with private mapping, precise mode and mmap memory\n" ); +evaluate_test (check_child_hugetlb_memory_mapping (USE_MMAP ,MTE_ASYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check child hugetlb memory with private mapping, precise mode and mmap memory\n" ); +evaluate_test (check_child_hugetlb_memory_mapping (USE_MPROTECT ,MTE_SYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check child hugetlb memory with private mapping, precise mode and mmap/mprotect memory\n" ); +evaluate_test (check_child_hugetlb_memory_mapping (USE_MPROTECT ,MTE_ASYNC_ERR ,MAP_PRIVATE |MAP_HUGETLB ), +"Check child hugetlb memory with private mapping, precise mode and mmap/mprotect memory\n" ); + +mte_restore_setup (); +free_hugetlb (); +ksft_print_cnts (); +return ksft_get_fail_cnt () ==0 ?KSFT_PASS :KSFT_FAIL ; +}
Signature · 1 line
2.41.0
Quoted 51 lines · Yang Shi
Enable MTE support for hugetlb.
The MTE page flags will be set on the folio only. When copying hugetlb folio (for example, CoW), the tags for all subpages will be copied when copying the first subpage.
When freeing hugetlb folio, the MTE flags will be cleared.
Signed-off-by Yang Shi <[email protected]>--- arch/arm64/include/asm/hugetlb.h | 15 ++++++- arch/arm64/include/asm/mman.h | 3 +- arch/arm64/include/asm/mte.h | 67 ++++++++++++++++++++++++++++++++ arch/arm64/kernel/hibernate.c | 7 ++++ arch/arm64/kernel/mte.c | 25 +++++++++++- arch/arm64/kvm/guest.c | 16 ++++++-- arch/arm64/kvm/mmu.c | 11 ++++++ arch/arm64/mm/copypage.c | 33 +++++++++++++--- fs/hugetlbfs/inode.c | 2 +- 9 files changed, 166 insertions(+), 13 deletions(-)v3: * Fixed the build error when !CONFIG_ARM64_MTE.
* Incorporated the comment from David to have hugetlb folio specific APIs for manipulating the page flags.
* Don't assume the first page is the head page since huge page copy can start from any subpage.
v2: * Reimplemented the patch to fix the comments from Catalin.
* Added test cases (patch #2) per Catalin.arch/arm64/include/asm/hugetlb.h +7
-0@@ -11,6 +11,7 @@#define __ASM_HUGETLB_H#include <asm/cacheflush.h> +#include <asm/mte.h> #include <asm/page.h> #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION @@ -18,9 +19,21 @@extern bool arch_hugetlb_migration_supported (struct hstate *h);#endif +#ifdef CONFIG_ARM64_MTE +#define CLEAR_FLAGS (BIT(PG_dcache_clean) | BIT(PG_mte_tagged) | \ + BIT(PG_mte_lock)) +#else +#define CLEAR_FLAGS BIT(PG_dcache_clean) +#endif
arch/arm64/include/asm/hugetlb.h line 36
Quoted 10 lines
arch/arm64/include/asm/hugetlb.h +7
-1+static inline void arch_clear_hugetlb_flags (struct folio *folio) { -clear_bit (PG_dcache_clean, &folio->flags); +if (!system_supports_mte ()) { +clear_bit (PG_dcache_clean, &folio->flags); +return ; + } + + folio->flags &= ~CLEAR_FLAGS ;
In contrast to clear_bit, this is now not an atomic operation anymore. Could we have concurrent modifications (locking the folio? mte?) where we could mess up (IOW, is there a reason we don't do __clear_bit in existing code)?
Maybe start with:
static inline void arch_clear_hugetlb_flags(struct folio *folio)
{
clear_bit(PG_dcache_clean, &folio->flags);
#ifdef CONFIG_ARM64_MTE
if (system_supports_mte()) {
clear_bit(PG_mte_tagged, &folio->flags);
clear_bit(PG_mte_lock, &folio->flags);
}#endif
}
And if you can argue that atomics are not required, convert all to __clear_bit() and have the compiler optimize it for you.
Quoted 33 lines
arch/arm64/include/asm/hugetlb.h +0
-0}#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags > diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.hindex
0f84518632b4..cec9fb6fec3b100644arch/arm64/include/asm/mte.h +6
-0@@ -41,6 +41,8 @@ void mte_free_tag_storage(char *storage);static inline void set_page_mte_tagged (struct page *page) { +VM_WARN_ON_ONCE (folio_test_hugetlb (page_folio (page))); +/* * Ensure that the tags written prior to this function are visible * before the page flags update. @@ -51,6 +53,8 @@ static inline void set_page_mte_tagged(struct page *page)static inline bool page_mte_tagged (struct page *page) { +VM_WARN_ON_ONCE (folio_test_hugetlb (page_folio (page))); +bool ret =test_bit (PG_mte_tagged, &page->flags);/* @@ -76,6 +80,8 @@ static inline bool page_mte_tagged(struct page *page)*/ static inline bool try_page_mte_tagging (struct page *page) { +VM_WARN_ON_ONCE (folio_test_hugetlb (page_folio (page))); +if (!test_and_set_bit (PG_mte_lock, &page->flags))return true;
[...]
arch/arm64/include/asm/mte.h static inline int mte_ptrace_copy_tags(struct task_struct *child, · line 224
Quoted 13 lines
arch/arm64/include/asm/mte.h +13
-0+static inline void set_folio_hugetlb_mte_tagged (struct folio *folio) +{ +} + +static inline bool folio_hugetlb_mte_tagged (struct folio *folio) +{ +return false; +} + +static inline bool try_folio_hugetlb_mte_tagging (struct folio *folio) +{ +return false; +}
I would suggest to stick to the format of our folio_test / folio_set ... functions. Please refer to folio_set_hugetlb_migratable/folio_set_hugetlb_hwpoison/ ...
Something like:
folio_test_hugetlb_mte_taggedfolio_set_hugetlb_mte_tagged
But the semantics of try_folio_hugetlb_mte_tagging() are a bit less obvious. I would suggest
folio_test_and_set_hugetlb_mte_lock()
We should probably clean up the page_* variants separately.
But ARM maintainers can feel free to intervene here.
Quoted 10 lines
arch/arm64/include/asm/mte.h +2
-0+#endif +static inline void mte_disable_tco_entry (struct task_struct *task) {if (!system_supports_mte ())arch/arm64/kernel/hibernate.c +0
-0
arch/arm64/kernel/hibernate.c static int swsusp_mte_save_tags(void) · line 273
Quoted 8 lines
arch/arm64/kernel/hibernate.c +1
-0@@ -266,10 +266,17 @@ static int swsusp_mte_save_tags(void) max_zone_pfn =zone_end_pfn (zone);for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {struct page *page =pfn_to_online_page (pfn); +struct folio *folio;if (!page)continue ;
Nit: I would drop this empty line.
Quoted 14 lines
arch/arm64/kernel/hibernate.c +6
-0+ folio =page_folio (page); + +if (folio_test_hugetlb (folio) && + !folio_hugetlb_mte_tagged (folio)) +continue ; +if (!page_mte_tagged (page))continue ;arch/arm64/kernel/mte.c +0
-0
arch/arm64/kernel/mte.c EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); · line 45
Quoted 9 lines
arch/arm64/kernel/mte.c +5
-1@@ -38,7 +38,22 @@ EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);void mte_sync_tags (pte_t pte,unsigned int nr_pages) {struct page *page =pte_page (pte); -unsigned int i; +struct folio *folio =page_folio (page); +unsigned long i; + +if (folio_test_hugetlb (folio)) { +unsigned long nr =folio_nr_pages (folio);
Nit: empty line please.
arch/arm64/kernel/mte.c EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); · line 53
Quoted 8 lines
arch/arm64/kernel/mte.c +8
-0+/* Hugetlb MTE flags are set for head page only */ +if (try_folio_hugetlb_mte_tagging (folio)) { +for (i =0 ; i < nr; i++, page++) +mte_clear_page_tags (page_address (page)); +set_folio_hugetlb_mte_tagged (folio); + } + +smp_wmb ();
We already do have one in set_folio_hugetlb_mte_tagged() [and try_folio_hugetlb_mte_tagging() does some magic as well], do we really need this smp_wmb()?
In general, I think checkpatch will tell you to document memory barriers and their counterparts thoroughly.
Quoted 3 lines
arch/arm64/kernel/mte.c +3
-0+ +return ; + }
Signature · 3 lines
Cheers, David / dhildenb
Quoted 53 lines · David Hildenbrand
Quoted 52 lines · Yang Shi
Enable MTE support for hugetlb.
The MTE page flags will be set on the folio only. When copying hugetlb folio (for example, CoW), the tags for all subpages will be copied when copying the first subpage.
When freeing hugetlb folio, the MTE flags will be cleared.
Signed-off-by Yang Shi <[email protected]>---
arch/arm64/include/asm/hugetlb.h| 15 ++++++- arch/arm64/include/asm/mman.h | 3 +- arch/arm64/include/asm/mte.h | 67 ++++++++++++++++++++++++++++++++ arch/arm64/kernel/hibernate.c | 7 ++++ arch/arm64/kernel/mte.c | 25 +++++++++++- arch/arm64/kvm/guest.c | 16 ++++++-- arch/arm64/kvm/mmu.c | 11 ++++++ arch/arm64/mm/copypage.c | 33 +++++++++++++--- fs/hugetlbfs/inode.c | 2 +-9 files changed, 166 insertions(+), 13 deletions(-)v3: * Fixed the build error when !CONFIG_ARM64_MTE.
* Incorporated the comment from David to have hugetlb folio specific APIs for manipulating the page flags.
* Don't assume the first page is the head page since huge page copy can start from any subpage.
v2: * Reimplemented the patch to fix the comments from Catalin.
* Added test cases (patch #2) per Catalin.arch/arm64/include/asm/hugetlb.h +1
-0@@ -11,6 +11,7 @@ #define __ASM_HUGETLB_H #include <asm/cacheflush.h> +#include <asm/mte.h> #include <asm/page.h> #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION@@ -18,9 +19,21 @@ extern bool arch_hugetlb_migration_supported(struct hstate *h); #endif+#ifdef
CONFIG_ARM64_MTEarch/arm64/include/asm/hugetlb.h +6
-0+#define CLEAR_FLAGS (BIT(PG_dcache_clean) | BIT(PG_mte_tagged) | \ + BIT(PG_mte_lock)) +#else +#define CLEAR_FLAGS BIT(PG_dcache_clean) +#endif +static inline void
arch_clear_hugetlb_flags(struct folio *folio){
-clear_bit(PG_dcache_clean, &folio->flags);
Quoted 8 lines
Quoted 6 lines
+ if (!system_supports_mte()) {
+clear_bit(PG_dcache_clean, &folio->flags); + return;
+ }
+
+folio->flags&= ~CLEAR_FLAGS;In contrast to
clear_bit, this is now not an atomic operation anymore. Could we have concurrent modifications (locking the folio? mte?) where we could mess up (IOW, is there a reason we don't do__clear_bitin existing code)?
AFAICT, I don't see there is any concurrent modification to them. arch_clear_hugetlb_flags() is called when:
* hugetlb folio is freed
* new hugetlb folio is enqueued (not available for allocation yet)
And is is protected by hugetlb_lock, all hugetlb folio allocation is serialized by the same lock. PG_mte_* flags can't be set before hugetlb folio is allocated. So I don't think atomic operation is actually necessary. Not sure if I miss something or not.
But all arches use clear_bit(). As you suggested below, it may be safer start with clear_bit().
Quoted 65 lines
Maybe start with:
static inline void
arch_clear_hugetlb_flags(struct folio *folio){
clear_bit(PG_dcache_clean,&folio->flags);
#ifdefCONFIG_ARM64_MTE
if (system_supports_mte()) {clear_bit(PG_mte_tagged, &folio->flags);clear_bit(PG_mte_lock, &folio->flags);
}
#endif
}And if you can argue that atomics are not required, convert all to
__clear_bit()and have the compiler optimize it for you.Quoted 30 lines
}
#definearch_clear_hugetlb_flagsarch_clear_hugetlb_flags
> diff --gita/arch/arm64/include/asm/mte.hb/arch/arm64/include/asm/mte.hindex0f84518632b4..cec9fb6fec3b100644arch/arm64/include/asm/mte.h +6
-0@@ -41,6 +41,8 @@ void mte_free_tag_storage(char *storage); static inline void set_page_mte_tagged(struct page *page) { + VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page))); + /** Ensure that the tags written prior to this function are visible * before the page flags update. @@ -51,6 +53,8 @@ static inline void set_page_mte_tagged(struct page *page) static inline bool page_mte_tagged(struct page *page) { + VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page))); + bool ret = test_bit(PG_mte_tagged, &page->flags); /*@@ -76,6 +80,8 @@ static inline bool page_mte_tagged(struct page *page)*/ static inline bool try_page_mte_tagging(struct page *page) { + VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page))); + if (!test_and_set_bit(PG_mte_lock, &page->flags)) return true;[...]
arch/arm64/include/asm/mte.h static inline int mte_ptrace_copy_tags(struct task_struct *child, · line 224
Quoted 13 lines
arch/arm64/include/asm/mte.h +13
-0+static inline void set_folio_hugetlb_mte_tagged (struct folio *folio) +{ +} + +static inline bool folio_hugetlb_mte_tagged (struct folio *folio) +{ + return false; +} + +static inline bool try_folio_hugetlb_mte_tagging (struct folio *folio) +{ + return false; +}I would suggest to stick to the format of our
folio_test/folio_set... functions. Please refer to folio_set_hugetlb_migratable/folio_set_hugetlb_hwpoison/ ...
Quoted 4 lines
Something like:
folio_test_hugetlb_mte_taggedfolio_set_hugetlb_mte_tagged
I don't have strong opinion on these two. They are straightforward enough and the users can easily map them to the page_* variants.
Quoted 4 lines
But the semantics of
try_folio_hugetlb_mte_tagging()are a bit less obvious. I would suggest
folio_test_and_set_hugetlb_mte_lock()
This one is a little bit hard to map to try_page_mte_tagging() TBH.
Quoted 3 lines
We should probably clean up the page_* variants separately.
Probably. The most confusing one is actually try_page_mte_tagging(). We should probably change it to test_and_set_page_mte_lock(). But anyway they are just used by arm64, so I don't worry too much and I'd like to hear from arm64 maintainers.
Quoted 13 lines
But ARM maintainers can feel free to intervene here.
Quoted 10 lines
arch/arm64/include/asm/mte.h +2
-0+#endif +static inline void
mte_disable_tco_entry(struct task_struct *task){
if (!system_supports_mte())arch/arm64/kernel/hibernate.c +0
-0@@ -266,10 +266,17 @@ static int swsusp_mte_save_tags(void)
Quoted 8 lines
arch/arm64/kernel/hibernate.c static int swsusp_mte_save_tags(void) · line 272
Quoted 6 lines
arch/arm64/kernel/hibernate.c +1
-0max_zone_pfn = zone_end_pfn(zone); for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { struct page *page = pfn_to_online_page(pfn); + struct folio *folio; if (!page) continue;Nit: I would drop this empty line.
OK
Quoted 17 lines
Quoted 17 lines
arch/arm64/kernel/hibernate.c +6
-0+ folio = page_folio(page); + + if (folio_test_hugetlb(folio) && + !folio_hugetlb_mte_tagged(folio)) + continue; + if (!page_mte_tagged(page)) continue;diff --git
a/arch/arm64/kernel/mte.cb/arch/arm64/kernel/mte.cindex6174671be7c1..c8b13bf36fc6100644arch/arm64/kernel/mte.c +0
-0@@ -38,7 +38,22 @@ EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); void mte_sync_tags(pte_t pte, unsigned int nr_pages) { struct page *page = pte_page(pte);
Quoted 8 lines
arch/arm64/kernel/mte.c EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); · line 45
Quoted 6 lines
arch/arm64/kernel/mte.c +5
-1- unsigned int i; + struct folio *folio = page_folio(page); + unsigned long i; + + if (folio_test_hugetlb(folio)) { + unsigned long nr = folio_nr_pages(folio);Nit: empty line please.
OK
Quoted 10 lines
arch/arm64/kernel/mte.c EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode); · line 53
Quoted 9 lines
arch/arm64/kernel/mte.c +8
-0+ /* Hugetlb MTE flags are set for head page only */ + if (try_folio_hugetlb_mte_tagging(folio)) { + for (i = 0; i < nr; i++, page++) + mte_clear_page_tags(page_address(page)); + set_folio_hugetlb_mte_tagged(folio); + } + + smp_wmb();
Quoted 3 lines
We already do have one in
set_folio_hugetlb_mte_tagged()[andtry_folio_hugetlb_mte_tagging()does some magic as well], do we really need thissmp_wmb()?In general, I think checkpatch will tell you to document memory barriers and their counterparts thoroughly.
It is not used to guarantee the PG_mte_tagged flag is visible, it is used to guarantee the tags are visible before PTE is set. The non-hugetlb path has:
/* ensure the tags are visible before the PTE is set */ smp_wmb();
I should keep the comment for hugetlb path as well.
Quoted 8 lines
Quoted 4 lines
arch/arm64/kernel/mte.c +3
-0+ + return; + }