Introduce cpu_icache_is_aliasing() across all architectures
Subsystems (4)
Zi Yan
reviewerPATCH v3 1/2 Introduce cpu_icache_is_aliasing() across all architectures
In commit eacd0e950dc2 ("ARC: [mm] Lazy D-cache flush (non aliasing VIPT)"), arc adds the need to flush dcache to make icache see the code page change. This also requires special handling for clear_user_(high)page(). Introduce cpu_icache_is_aliasing() to make MM code query special clear_user_(high)page() easier. This will be used by the following commit.
--- arch/arc/Kconfig | 1 + arch/arc/include/asm/cachetype.h | 8 ++++++++ include/linux/cacheinfo.h | 6 ++++++ 3 files changed, 15 insertions(+) create mode 100644 arch/arc/include/asm/cachetype.h
arch/arc/Kconfig +1 -0
@@ -6,6 +6,7 @@ config ARC def_bool y select ARC_TIMERS + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DMA_PREP_COHERENT
arch/arc/include/asm/cachetype.h new file +8 -0
@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ARC_CACHETYPE_H +#define __ASM_ARC_CACHETYPE_H + +#define cpu_dcache_is_aliasing () false +#define cpu_icache_is_aliasing () true + +#endif
include/linux/cacheinfo.h +6 -0
@@ -155,8 +155,14 @@ static inline int get_cpu_cacheinfo_id(int cpu, int level)#ifndef CONFIG_ARCH_HAS_CPU_CACHE_ALIASING #define cpu_dcache_is_aliasing () false +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing()#else #include <asm/cachetype.h> + +#ifndef cpu_icache_is_aliasing +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing() +#endif +#endif #endif /* _LINUX_CACHEINFO_H */
Signature · 1 line
2.45.2
Zi Yan
reviewerPATCH v3 2/2 mm: use clear_user_(high)page() for arch with special user folio handling
For architectures setting cpu_dcache_is_aliasing() to true, which require flushing dcache, and arc setting cpu_icache_is_aliasing() to true changes folio->flags to make icache coherent to dcache after clearing a user folio, __GFP_ZERO using only clear_page() is not enough to zero user folios and clear_user_(high)page() must be used. Otherwise, user data will be corrupted.
Fix it by always clearing user folios with clear_user_(high)page() when cpu_dcache_is_aliasing() is true or cpu_icache_is_aliasing() is true. Rename alloc_zeroed() to alloc_need_zeroing() and invert the logic to clarify its intend.
--- include/linux/highmem.h | 8 +++++++- include/linux/mm.h | 18 ++++++++++++++++++ mm/huge_memory.c | 9 +++++---- mm/internal.h | 6 ------ mm/memory.c | 10 +++++----- 5 files changed, 35 insertions(+), 16 deletions(-)
include/linux/highmem.h +7 -1
@@ -224,7 +224,13 @@ static inlinestruct folio *vma_alloc_zeroed_movable_folio (struct vm_area_struct *vma,unsigned long vaddr) { -return vma_alloc_folio (GFP_HIGHUSER_MOVABLE | __GFP_ZERO,0 , vma, vaddr); +struct folio *folio; + + folio =vma_alloc_folio (GFP_HIGHUSER_MOVABLE ,0 , vma, vaddr); +if (folio &&alloc_need_zeroing ()) +clear_user_highpage (&folio->page, vaddr); + +return folio; }#endif
include/linux/mm.h +18 -0
@@ -31,6 +31,7 @@#include <linux/kasan.h> #include <linux/memremap.h> #include <linux/slab.h> +#include <linux/cacheinfo.h> struct mempolicy ;struct anon_vma ;
@@ -4175,6 +4176,23 @@ static inline int do_mseal(unsigned long start, size_t len_in, unsigned long fla }#endif +/* +* alloc_need_zeroing checks if a user folio from page allocator needs to be +* zeroed or not. +*/ +static inline bool alloc_need_zeroing (void ) +{ +/* +* for user folios, arch with cache aliasing requires cache flush and +* arc changes folio->flags to make icache coherent with dcache, so +* always return false to make caller use +* clear_user_page()/clear_user_highpage() +*/ +return (cpu_dcache_is_aliasing () ||cpu_icache_is_aliasing ()) || + !static_branch_maybe (CONFIG_INIT_ON_ALLOC_DEFAULT_ON , + &init_on_alloc); +} +int arch_get_shadow_stack_status (struct task_struct *t,unsigned long __user *status);int arch_set_shadow_stack_status (struct task_struct *t,unsigned long status);int arch_lock_shadow_stack_status (struct task_struct *t,unsigned long status);
mm/huge_memory.c +5 -4
@@ -1176,11 +1176,12 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,folio_throttle_swaprate (folio, gfp);/* -* When a folio is not zeroed during allocation (__GFP_ZERO not used), -* folio_zero_user() is used to make sure that the page corresponding -* to the faulting address will be hot in the cache after zeroing. +* When a folio is not zeroed during allocation (__GFP_ZERO not used) +* or user folios require special handling, folio_zero_user() is used to +* make sure that the page corresponding to the faulting address will be +* hot in the cache after zeroing. */ -if (!alloc_zeroed ()) +if (alloc_need_zeroing ())folio_zero_user (folio, addr);/* * The memory barrier inside __folio_mark_uptodate makes sure that
mm/internal.h +0 -6
@@ -1285,12 +1285,6 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,void touch_pmd (struct vm_area_struct *vma,unsigned long addr,pmd_t *pmd,bool write); -static inline bool alloc_zeroed (void ) -{ -return static_branch_maybe (CONFIG_INIT_ON_ALLOC_DEFAULT_ON , - &init_on_alloc); -} -/* * Parses a string with mem suffixes into its order. Useful to parse kernel * parameters.
mm/memory.c +5 -5
@@ -4733,12 +4733,12 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf)folio_throttle_swaprate (folio, gfp);/* * When a folio is not zeroed during allocation -* (__GFP_ZERO not used), folio_zero_user() is used -* to make sure that the page corresponding to the -* faulting address will be hot in the cache after -* zeroing. +* (__GFP_ZERO not used) or user folios require special +* handling, folio_zero_user() is used to make sure +* that the page corresponding to the faulting address +* will be hot in the cache after zeroing. */ -if (!alloc_zeroed ()) +if (alloc_need_zeroing ())folio_zero_user (folio, vmf->address);return folio; }
Signature · 1 line
2.45.2
Re: [PATCH v3 2/2] mm: use clear_user_(high)page() for arch with special user folio handling
Quoted 5 lines · Zi Yan
For architectures setting
cpu_dcache_is_aliasing()to true, which require flushing dcache, and arc settingcpu_icache_is_aliasing()to true changesfolio->flagsto make icache coherent to dcache after clearing a user folio,__GFP_ZEROusing onlyclear_page()is not enough to zero user folios and clear_user_(high)page() must be used.
The sentence above is rather long and unclear, perhaps split it into multiple simpler sentences to clarify the intent ?
Quoted 57 lines
Otherwise, user data
will be corrupted.Fix it by always clearing user folios with clear_user_(high)page() when
cpu_dcache_is_aliasing()is true orcpu_icache_is_aliasing()is true. Renamealloc_zeroed()toalloc_need_zeroing()and invert the logic to clarify its intend.Fixes 5708d96da20b ("mm: avoid zeroing user movable page twice with init_on_alloc=1")Reported-by Geert Uytterhoeven <[email protected]>Closes https://lore.kernel.org/linux-mm/CAMuHMdV1hRp_NtR5YnJo=HsfgKQeH91J537Gh4gKk3PFZhSkbA@mail.gmail.com/ open hereTested-by Geert Uytterhoeven <[email protected]>Signed-off-by Zi Yan <[email protected]>--- include/linux/highmem.h | 8 +++++++- include/linux/mm.h | 18 ++++++++++++++++++ mm/huge_memory.c | 9 +++++---- mm/internal.h | 6 ------ mm/memory.c | 10 +++++----- 5 files changed, 35 insertions(+), 16 deletions(-)include/linux/highmem.h +7
-1@@ -224,7 +224,13 @@ static inlinestruct folio *vma_alloc_zeroed_movable_folio (struct vm_area_struct *vma,unsigned long vaddr) { -return vma_alloc_folio (GFP_HIGHUSER_MOVABLE | __GFP_ZERO,0 , vma, vaddr); +struct folio *folio; + + folio =vma_alloc_folio (GFP_HIGHUSER_MOVABLE ,0 , vma, vaddr); +if (folio &&alloc_need_zeroing ()) +clear_user_highpage (&folio->page, vaddr); + +return folio; }#endif include/linux/mm.h +2
-0@@ -31,6 +31,7 @@#include <linux/kasan.h> #include <linux/memremap.h> #include <linux/slab.h> +#include <linux/cacheinfo.h> struct mempolicy ;struct anon_vma ;@@ -4175,6 +4176,23 @@ static inline int do_mseal(unsigned long start, size_t len_in, unsigned long fla }#endif +/*
include/linux/mm.h static inline int do_mseal(unsigned long start, size_t len_in, unsigned long fla · line 4189
Quoted 10 lines
include/linux/mm.h +10
-0+* alloc_need_zeroing checks if a user folio from page allocator needs to be +* zeroed or not. +*/ +static inline bool alloc_need_zeroing (void ) +{ +/* +* for user folios, arch with cache aliasing requires cache flush and +* arc changes folio->flags to make icache coherent with dcache, so +* always return false to make caller use +* clear_user_page()/clear_user_highpage()
Missing "." at the end of comment.
include/linux/mm.h static inline int do_mseal(unsigned long start, size_t len_in, unsigned long fla · line 4191
Quoted 2 lines
include/linux/mm.h +2
-0+*/ + return (cpu_dcache_is_aliasing() || cpu_icache_is_aliasing()) ||
I suspect the () around "cpu_dcache_is_aliasing() || cpu_icache_is_aliasing()" is not needed.
Thanks,
Mathieu
Quoted 67 lines
include/linux/mm.h +4
-0+ !static_branch_maybe (CONFIG_INIT_ON_ALLOC_DEFAULT_ON , + &init_on_alloc); +} +int arch_get_shadow_stack_status (struct task_struct *t,unsigned long __user *status);int arch_set_shadow_stack_status (struct task_struct *t,unsigned long status);int arch_lock_shadow_stack_status (struct task_struct *t,unsigned long status);mm/huge_memory.c +5
-4@@ -1176,11 +1176,12 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,folio_throttle_swaprate (folio, gfp);/* -* When a folio is not zeroed during allocation (__GFP_ZERO not used), -* folio_zero_user() is used to make sure that the page corresponding -* to the faulting address will be hot in the cache after zeroing. +* When a folio is not zeroed during allocation (__GFP_ZERO not used) +* or user folios require special handling, folio_zero_user() is used to +* make sure that the page corresponding to the faulting address will be +* hot in the cache after zeroing. */ -if (!alloc_zeroed ()) +if (alloc_need_zeroing ())folio_zero_user (folio, addr);/* * The memory barrier inside __folio_mark_uptodate makes sure that mm/internal.h +0
-6@@ -1285,12 +1285,6 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,void touch_pmd (struct vm_area_struct *vma,unsigned long addr,pmd_t *pmd,bool write); -static inline bool alloc_zeroed (void ) -{ -return static_branch_maybe (CONFIG_INIT_ON_ALLOC_DEFAULT_ON , - &init_on_alloc); -} -/* * Parses a string with mem suffixes into its order. Useful to parse kernel * parameters. mm/memory.c +5
-5@@ -4733,12 +4733,12 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf)folio_throttle_swaprate (folio, gfp);/* * When a folio is not zeroed during allocation -* (__GFP_ZERO not used), folio_zero_user() is used -* to make sure that the page corresponding to the -* faulting address will be hot in the cache after -* zeroing. +* (__GFP_ZERO not used) or user folios require special +* handling, folio_zero_user() is used to make sure +* that the page corresponding to the faulting address +* will be hot in the cache after zeroing. */ -if (!alloc_zeroed ()) +if (alloc_need_zeroing ())folio_zero_user (folio, vmf->address);return folio; }
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Quoted 9 lines · Zi Yan
In commit
eacd0e950dc2("ARC: [mm] Lazy D-cache flush (non aliasing VIPT)"), arc adds the need to flush dcache to make icache see the code page change. This also requires special handling for clear_user_(high)page(). Introducecpu_icache_is_aliasing()to make MM code query special clear_user_(high)page() easier. This will be used by the following commit.Suggested-by Mathieu Desnoyers <[email protected]>Signed-off-by Zi Yan <[email protected]>
Quoted 52 lines
--- arch/arc/Kconfig | 1 + arch/arc/include/asm/cachetype.h | 8 ++++++++ include/linux/cacheinfo.h | 6 ++++++ 3 files changed, 15 insertions(+)create mode 100644
arch/arc/include/asm/cachetype.harch/arc/Kconfig +1
-0@@ -6,6 +6,7 @@ config ARC def_bool y select ARC_TIMERS + select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DMA_PREP_COHERENTarch/arc/include/asm/cachetype.h new file +8
-0@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ARC_CACHETYPE_H +#define __ASM_ARC_CACHETYPE_H + +#define cpu_dcache_is_aliasing () false +#define cpu_icache_is_aliasing () true + +#endif include/linux/cacheinfo.h +6
-0@@ -155,8 +155,14 @@ static inline int get_cpu_cacheinfo_id(int cpu, int level)#ifndef CONFIG_ARCH_HAS_CPU_CACHE_ALIASING #define cpu_dcache_is_aliasing () false +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing()#else #include <asm/cachetype.h> + +#ifndef cpu_icache_is_aliasing +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing() +#endif +#endif #endif /* _LINUX_CACHEINFO_H */
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com
Quoted 8 lines · Mathieu Desnoyers
Quoted 7 lines · Zi Yan
In commit
eacd0e950dc2("ARC: [mm] Lazy D-cache flush (non aliasing VIPT)"), arc adds the need to flush dcache to make icache see the code page change. This also requires special handling for clear_user_(high)page(). Introducecpu_icache_is_aliasing()to make MM code query special clear_user_(high)page() easier. This will be used by the following commit.
Quoted 5 lines
Quoted 2 lines
Suggested-by Mathieu Desnoyers <[email protected]>Signed-off-by Zi Yan <[email protected]>Reviewed-by Mathieu Desnoyers <[email protected]>
We should probably use this new cpu_icache_is_aliasing() to gate availability of DAX XIP, as described in the commit message here:
commit 8690bbcf3b ("Introduce cpu_dcache_is_aliasing() across all architectures)"
Note that this leaves "cpu_icache_is_aliasing()" to be implemented as future work. This would be useful to gate features like XIP on architectures which have aliasing CPU dcache-icache but not CPU dcache-dcache.
Thanks,
Mathieu
Quoted 52 lines
Quoted 51 lines
---
arch/arc/Kconfig | 1 +
arch/arc/include/asm/cachetype.h| 8 ++++++++
include/linux/cacheinfo.h | 6 ++++++3 files changed, 15 insertions(+)create mode 100644
arch/arc/include/asm/cachetype.harch/arc/Kconfig +0
-0@@ -6,6 +6,7 @@config ARC
def_booly
selectARC_TIMERS
+ selectARCH_HAS_CPU_CACHE_ALIASING
selectARCH_HAS_CACHE_LINE_SIZE
selectARCH_HAS_DEBUG_VM_PGTABLE
selectARCH_HAS_DMA_PREP_COHERENTarch/arc/include/asm/cachetype.h new file +8
-0@@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ARC_CACHETYPE_H +#define __ASM_ARC_CACHETYPE_H + +#define cpu_dcache_is_aliasing () false +#define cpu_icache_is_aliasing () true + +#endif include/linux/cacheinfo.h +6
-0@@ -155,8 +155,14 @@ static inline int get_cpu_cacheinfo_id(int cpu, int level) #ifndef CONFIG_ARCH_HAS_CPU_CACHE_ALIASING #define cpu_dcache_is_aliasing() false +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing() #else #include <asm/cachetype.h> + +#ifndef cpu_icache_is_aliasing +#define cpu_icache_is_aliasing () cpu_dcache_is_aliasing() +#endif + #endif #endif /* _LINUX_CACHEINFO_H */
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. https://www.efficios.com