libnvdimm: Add prctl control for disabling synchronous fault support.
Subsystems (7)
RFC PATCH 1/2 libnvdimm: Add prctl control for disabling synchronous fault support.
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of MAP_SYNC if applications are not using the new instructions on newer hardware.
This patch adds a prctl option MAP_SYNC_ENABLE that can be used to enable the usage of MAP_SYNC. The kernel config option is added to allow the user to control whether MAP_SYNC should be enabled by default or not.
--- include/linux/sched/coredump.h | 13 ++++++++++--- include/uapi/linux/prctl.h | 3 +++ kernel/fork.c | 8 +++++++- kernel/sys.c | 18 ++++++++++++++++++ mm/Kconfig | 3 +++ mm/mmap.c | 4 ++++ 6 files changed, 45 insertions(+), 4 deletions(-)
include/linux/sched/coredump.h +10 -3
@@ -72,9 +72,16 @@ static inline int get_dumpable(struct mm_struct *mm)#define MMF_DISABLE_THP 24/* disable THP for all VMAs */ #define MMF_OOM_VICTIM 25/* mm is the oom victim */ #define MMF_OOM_REAP_QUEUED 26/* mm was queued for oom_reaper */ -#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC 27/* disable THP for all VMAs */ +#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC_MASK (1 << MMF_DISABLE_MAP_SYNC) -#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ - MMF_DISABLE_THP_MASK) +#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK | \ + MMF_DISABLE_THP_MASK | MMF_DISABLE_MAP_SYNC_MASK) + +static inline bool map_sync_enabled (struct mm_struct *mm) +{ +return !(mm->flags &MMF_DISABLE_MAP_SYNC_MASK ); +}#endif /* _LINUX_SCHED_COREDUMP_H */
include/uapi/linux/prctl.h +3 -0
@@ -238,4 +238,7 @@ struct prctl_mm_map {#define PR_SET_IO_FLUSHER 57#define PR_GET_IO_FLUSHER 58 +#define PR_SET_MAP_SYNC_ENABLE 59 +#define PR_GET_MAP_SYNC_ENABLE 60 +#endif /* _LINUX_PRCTL_H */
kernel/fork.c +7 -1
@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ; +#endif +static int __init coredump_filter_setup (char *s) { default_dump_filter =
@@ -1039,7 +1045,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->flags = current->mm->flags &MMF_INIT_MASK ; mm->def_flags = current->mm->def_flags &VM_INIT_DEF_MASK ; }else { - mm->flags = default_dump_filter; + mm->flags = default_dump_filter | default_map_sync_mask; mm->def_flags =0 ; }
kernel/sys.c +18 -0
@@ -2450,6 +2450,24 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,clear_bit (MMF_DISABLE_THP , &me->mm->flags);up_write (&me->mm->mmap_sem);break ; + +case PR_GET_MAP_SYNC_ENABLE : +if (arg2 || arg3 || arg4 || arg5) +return -EINVAL ; + error = !test_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +break ; +case PR_SET_MAP_SYNC_ENABLE : +if (arg3 || arg4 || arg5) +return -EINVAL ; +if (down_write_killable (&me->mm->mmap_sem)) +return -EINTR ; +if (arg2) +clear_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +else +set_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +up_write (&me->mm->mmap_sem); +break ; +case PR_MPX_ENABLE_MANAGEMENT :case PR_MPX_DISABLE_MANAGEMENT :/* No longer implemented: */
mm/Kconfig +3 -0
@@ -867,4 +867,7 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +config ARCH_MAP_SYNC_DISABLE + bool + endmenu
mm/mmap.c +4 -0
@@ -1464,6 +1464,10 @@ unsigned long do_mmap(struct file *file, unsigned long addr,case MAP_SHARED_VALIDATE :if (flags & ~flags_mask)return -EOPNOTSUPP ; + +if ((flags &MAP_SYNC ) && !map_sync_enabled (mm)) +return -EOPNOTSUPP ; +if (prot &PROT_WRITE ) {if (!(file->f_mode &FMODE_WRITE ))return -EACCES ;
Signature · 4 lines
2.26.2 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
RFC PATCH 2/2 powerpc/pmem: Disable synchronous fault by default.
This adds a kernel config option that controls whether MAP_SYNC is enabled by default. With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of MAP_SYNC if applications are not using the new instructions on newer hardware.
This config allows user to control whether MAP_SYNC should be enabled by default or not.
--- arch/powerpc/platforms/Kconfig.cputype | 9 +++++++++ 1 file changed, 9 insertions(+)
arch/powerpc/platforms/Kconfig.cputype +9 -0
@@ -383,6 +383,15 @@ config PPC_KUEP If you're unsure, say Y. +config ARCH_MAP_SYNC_DISABLE + bool "Disable synchronous fault support (MAP_SYNC)" + default y + help + Disable support for synchronous fault with nvdimm namespaces. + + If you're unsure, say Y. + + config PPC_HAVE_KUAP bool
Signature · 4 lines
2.26.2 _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Adding Jan
Quoted 139 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>--- include/linux/sched/coredump.h | 13 ++++++++++--- include/uapi/linux/prctl.h | 3 +++ kernel/fork.c | 8 +++++++- kernel/sys.c | 18 ++++++++++++++++++ mm/Kconfig | 3 +++ mm/mmap.c | 4 ++++ 6 files changed, 45 insertions(+), 4 deletions(-)include/linux/sched/coredump.h +10
-3@@ -72,9 +72,16 @@ static inline int get_dumpable(struct mm_struct *mm)#define MMF_DISABLE_THP 24/* disable THP for all VMAs */ #define MMF_OOM_VICTIM 25/* mm is the oom victim */ #define MMF_OOM_REAP_QUEUED 26/* mm was queued for oom_reaper */ -#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC 27/* disable THP for all VMAs */ +#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC_MASK (1 << MMF_DISABLE_MAP_SYNC) -#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ - MMF_DISABLE_THP_MASK) +#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK | \ + MMF_DISABLE_THP_MASK | MMF_DISABLE_MAP_SYNC_MASK) + +static inline bool map_sync_enabled (struct mm_struct *mm) +{ +return !(mm->flags &MMF_DISABLE_MAP_SYNC_MASK ); +}#endif /* _LINUX_SCHED_COREDUMP_H */ include/uapi/linux/prctl.h +3
-0@@ -238,4 +238,7 @@ struct prctl_mm_map {#define PR_SET_IO_FLUSHER 57#define PR_GET_IO_FLUSHER 58 +#define PR_SET_MAP_SYNC_ENABLE 59 +#define PR_GET_MAP_SYNC_ENABLE 60 +#endif /* _LINUX_PRCTL_H */ kernel/fork.c +7
-1@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ; +#endif +static int __init coredump_filter_setup (char *s) { default_dump_filter =@@ -1039,7 +1045,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->flags = current->mm->flags &MMF_INIT_MASK ; mm->def_flags = current->mm->def_flags &VM_INIT_DEF_MASK ; }else { - mm->flags = default_dump_filter; + mm->flags = default_dump_filter | default_map_sync_mask; mm->def_flags =0 ; }kernel/sys.c +18
-0@@ -2450,6 +2450,24 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,clear_bit (MMF_DISABLE_THP , &me->mm->flags);up_write (&me->mm->mmap_sem);break ; + +case PR_GET_MAP_SYNC_ENABLE : +if (arg2 || arg3 || arg4 || arg5) +return -EINVAL ; + error = !test_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +break ; +case PR_SET_MAP_SYNC_ENABLE : +if (arg3 || arg4 || arg5) +return -EINVAL ; +if (down_write_killable (&me->mm->mmap_sem)) +return -EINTR ; +if (arg2) +clear_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +else +set_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +up_write (&me->mm->mmap_sem); +break ; +case PR_MPX_ENABLE_MANAGEMENT :case PR_MPX_DISABLE_MANAGEMENT :/* No longer implemented: */ mm/Kconfig +3
-0@@ -867,4 +867,7 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +config ARCH_MAP_SYNC_DISABLE + bool + endmenumm/mmap.c +4
-0@@ -1464,6 +1464,10 @@ unsigned long do_mmap(struct file *file, unsigned long addr,case MAP_SHARED_VALIDATE :if (flags & ~flags_mask)return -EOPNOTSUPP ; + +if ((flags &MAP_SYNC ) && !map_sync_enabled (mm)) +return -EOPNOTSUPP ; +if (prot &PROT_WRITE ) {if (!(file->f_mode &FMODE_WRITE ))return -EACCES ;Signature · 1 line
2.26.2
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Hi,
Thanks Michal. I also missed Jeff in this email thread.
-aneesh
Quoted 134 lines · Michal Suchánek
Adding Jan
Quoted 131 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>--- include/linux/sched/coredump.h | 13 ++++++++++--- include/uapi/linux/prctl.h | 3 +++ kernel/fork.c | 8 +++++++- kernel/sys.c | 18 ++++++++++++++++++ mm/Kconfig | 3 +++ mm/mmap.c | 4 ++++ 6 files changed, 45 insertions(+), 4 deletions(-)include/linux/sched/coredump.h +10
-3@@ -72,9 +72,16 @@ static inline int get_dumpable(struct mm_struct *mm)#define MMF_DISABLE_THP 24/* disable THP for all VMAs */ #define MMF_OOM_VICTIM 25/* mm is the oom victim */ #define MMF_OOM_REAP_QUEUED 26/* mm was queued for oom_reaper */ -#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC 27/* disable THP for all VMAs */ +#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) +#define MMF_DISABLE_MAP_SYNC_MASK (1 << MMF_DISABLE_MAP_SYNC) -#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ - MMF_DISABLE_THP_MASK) +#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK | \ + MMF_DISABLE_THP_MASK | MMF_DISABLE_MAP_SYNC_MASK) + +static inline bool map_sync_enabled (struct mm_struct *mm) +{ +return !(mm->flags &MMF_DISABLE_MAP_SYNC_MASK ); +}#endif /* _LINUX_SCHED_COREDUMP_H */ include/uapi/linux/prctl.h +3
-0@@ -238,4 +238,7 @@ struct prctl_mm_map {#define PR_SET_IO_FLUSHER 57#define PR_GET_IO_FLUSHER 58 +#define PR_SET_MAP_SYNC_ENABLE 59 +#define PR_GET_MAP_SYNC_ENABLE 60 +#endif /* _LINUX_PRCTL_H */ kernel/fork.c +7
-1@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ; +#endif +static int __init coredump_filter_setup (char *s) { default_dump_filter =@@ -1039,7 +1045,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->flags = current->mm->flags &MMF_INIT_MASK ; mm->def_flags = current->mm->def_flags &VM_INIT_DEF_MASK ; }else { - mm->flags = default_dump_filter; + mm->flags = default_dump_filter | default_map_sync_mask; mm->def_flags =0 ; }kernel/sys.c +18
-0@@ -2450,6 +2450,24 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,clear_bit (MMF_DISABLE_THP , &me->mm->flags);up_write (&me->mm->mmap_sem);break ; + +case PR_GET_MAP_SYNC_ENABLE : +if (arg2 || arg3 || arg4 || arg5) +return -EINVAL ; + error = !test_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +break ; +case PR_SET_MAP_SYNC_ENABLE : +if (arg3 || arg4 || arg5) +return -EINVAL ; +if (down_write_killable (&me->mm->mmap_sem)) +return -EINTR ; +if (arg2) +clear_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +else +set_bit (MMF_DISABLE_MAP_SYNC , &me->mm->flags); +up_write (&me->mm->mmap_sem); +break ; +case PR_MPX_ENABLE_MANAGEMENT :case PR_MPX_DISABLE_MANAGEMENT :/* No longer implemented: */ mm/Kconfig +3
-0@@ -867,4 +867,7 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +config ARCH_MAP_SYNC_DISABLE + bool + endmenumm/mmap.c +2
-0@@ -1464,6 +1464,10 @@ unsigned long do_mmap(struct file *file, unsigned long addr,case MAP_SHARED_VALIDATE :if (flags & ~flags_mask)return -EOPNOTSUPP ; + +if ((flags &MAP_SYNC ) && !map_sync_enabled (mm))
Quoted 8 lines
Quoted 8 lines
mm/mmap.c +2
-0+return -EOPNOTSUPP ; +if (prot &PROT_WRITE ) {if (!(file->f_mode &FMODE_WRITE ))return -EACCES ;Signature · 1 line
2.26.2
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>
...
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
kernel/fork.c +0
-0
Quoted 8 lines
Quoted 8 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 8 lines
kernel/fork.c +6
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ; +#endif +
I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable MAP_SYNC for all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initialize default_map_sync_mask on boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.
Honza
Signature · 5 lines
Jan Kara <[email protected]> SUSE Labs, CR _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 32 lines · Jan Kara
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 971
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.
yes that is correct. We ideally want to deny MAP_SYNC only w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disable MAP_SYNC if we find a device with the specific value.
BTW with the recent changes I posted for the nvdimm driver, older kernel won't initialize persistent memory device on newer hardware. Newer hardware will present the device to OS with a different device tree compat string.
My expectation w.r.t this patch was, Distro would want to markCONFIG_ARCH_MAP_SYNC_DISABLE=n based on the different application certification. Otherwise application will have to end up calling the prctl(MMF_DISABLE_MAP_SYNC, 0) any way. If that is the case, should this be dependent on P10?
With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 64 lines · Aneesh Kumar K.V
Quoted 32 lines · Jan Kara
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 971
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNCif we find a device with the specific value.BTW with the recent changes I posted for the nvdimm driver, older kernel won't initialize persistent memory device on newer hardware. Newer hardware will present the device to OS with a different device tree compat string.
My expectation w.r.t this patch was, Distro would want to mark
CONFIG_ARCH_MAP_SYNC_DISABLE=nbased on the different application certification. Otherwise application will have to end up calling theprctl(MMF_DISABLE_MAP_SYNC, 0)any way. If that is the case, should this be dependent on P10?With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
Quoted 6 lines
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
I think prctl is the wrong interface for this. I was thinking a sysfs interface along the same lines as /sys/block/pmemX/dax/write_cache. That attribute is toggling DAXDEV_WRITE_CACHE for the determination of whether the platform or the kernel needs to handle cache flushing relative to power loss. A similar attribute can be established for DAXDEV_SYNC, it would simply default to off based on a configuration time policy, but be dynamically changeable at runtime via sysfs.
These flags are device properties that affect the kernel and userspace's handling of persistence.
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 81 lines · Dan Williams
Quoted 64 lines · Aneesh Kumar K.V
Quoted 32 lines · Jan Kara
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 971
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNCif we find a device with the specific value.BTW with the recent changes I posted for the nvdimm driver, older kernel won't initialize persistent memory device on newer hardware. Newer hardware will present the device to OS with a different device tree compat string.
My expectation w.r.t this patch was, Distro would want to mark
CONFIG_ARCH_MAP_SYNC_DISABLE=nbased on the different application certification. Otherwise application will have to end up calling theprctl(MMF_DISABLE_MAP_SYNC, 0)any way. If that is the case, should this be dependent on P10?With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
Quoted 6 lines
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
I think prctl is the wrong interface for this. I was thinking a sysfs interface along the same lines as
/sys/block/pmemX/dax/write_cache. That attribute is togglingDAXDEV_WRITE_CACHEfor the determination of whether the platform or the kernel needs to handle cache flushing relative to power loss. A similar attribute can be established forDAXDEV_SYNC, it would simply default to off based on a configuration time policy, but be dynamically changeable at runtime via sysfs.
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
-aneeseh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 83 lines · Aneesh Kumar K.V
Quoted 81 lines · Dan Williams
Quoted 64 lines · Aneesh Kumar K.V
Quoted 32 lines · Jan Kara
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNCif we find a device with the specific value.BTW with the recent changes I posted for the nvdimm driver, older kernel won't initialize persistent memory device on newer hardware. Newer hardware will present the device to OS with a different device tree compat string.
My expectation w.r.t this patch was, Distro would want to mark
CONFIG_ARCH_MAP_SYNC_DISABLE=nbased on the different application certification. Otherwise application will have to end up calling theprctl(MMF_DISABLE_MAP_SYNC, 0)any way. If that is the case, should this be dependent on P10?With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
Quoted 6 lines
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
I think prctl is the wrong interface for this. I was thinking a sysfs interface along the same lines as
/sys/block/pmemX/dax/write_cache. That attribute is togglingDAXDEV_WRITE_CACHEfor the determination of whether the platform or the kernel needs to handle cache flushing relative to power loss. A similar attribute can be established forDAXDEV_SYNC, it would simply default to off based on a configuration time policy, but be dynamically changeable at runtime via sysfs.
Quoted 7 lines
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
_______________________________________________
Linux-nvdimm mailing list -- [email protected]
To unsubscribe send an email to [email protected]
Quoted 89 lines · Dan Williams
Quoted 87 lines · Aneesh Kumar K.V
Quoted 84 lines · Dan Williams
Quoted 64 lines · Aneesh Kumar K.V
Quoted 32 lines · Jan Kara
Hi!
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNCif we find a device with the specific value.BTW with the recent changes I posted for the nvdimm driver, older kernel won't initialize persistent memory device on newer hardware. Newer hardware will present the device to OS with a different device tree compat string.
My expectation w.r.t this patch was, Distro would want to mark
CONFIG_ARCH_MAP_SYNC_DISABLE=nbased on the different application certification. Otherwise application will have to end up calling theprctl(MMF_DISABLE_MAP_SYNC, 0)any way. If that is the case, should this be dependent on P10?With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
Quoted 6 lines
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
I think prctl is the wrong interface for this. I was thinking a sysfs interface along the same lines as
/sys/block/pmemX/dax/write_cache. That attribute is togglingDAXDEV_WRITE_CACHEfor the determination of whether the platform or the kernel needs to handle cache flushing relative to power loss. A similar attribute can be established forDAXDEV_SYNC, it would simply default to off based on a configuration time policy, but be dynamically changeable at runtime via sysfs.These flags are device properties that affect the kernel and userspace's handling of persistence.
Quoted 6 lines
Quoted 3 lines
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
First I'd like to mention that hopefully the concern is mostly theoretical since as Aneesh wrote above, real persistent memory never shipped for PPC and so there are very few apps (if any) using the old way to ensure cache flushing.
But I'd like to understand why do you think per-process control is a recipe for disaster? Because from my POV the sysfs interface you propose is actually difficult to use in practice. As a distributor, you have hard time picking the default because you have a choice between picking safe option which is going to confuse users because of failing MAP_SYNC and unsafe option where everyone will be happy until someone looses data because of some ancient application using wrong instructions to persist data. Poor experience for users in either way. And when distro defaults to "safe option", then the burden is on the sysadmin to toggle the switch but how is he supposed to decide when that is safe? First he has to understand what the problem actually is, then he has to audit all the applications using pmem whether they use the new instruction - which is IMO a lot of effort if you have a couple of applications and practically infeasible if you have more of them. So IMO the burden should be *on the application* to declare that it is aware of the new instructions to flush pmem on the platform and only to such application the kernel should give the trust to use MAP_SYNC mappings.
Honza
Signature · 5 lines
Jan Kara <[email protected]> SUSE Labs, CR _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
[ forgive formatting, a series of unfortunate events has me using Outlook for the moment ]
Quoted 17 lines
From: Jan Kara <[email protected]>
Quoted 10 lines
Quoted 7 lines
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
First I'd like to mention that hopefully the concern is mostly theoretical since as Aneesh wrote above, real persistent memory never shipped for PPC and so there are very few apps (if any) using the old way to ensure cache flushing.
Quoted 16 lines
But I'd like to understand why do you think per-process control is a recipe for disaster? Because from my POV the sysfs interface you propose is actually difficult to use in practice. As a distributor, you have hard time picking the default because you have a choice between picking safe option which is going to confuse users because of failing
MAP_SYNCand unsafe option where everyone will be happy until someone looses data because of some ancient application using wrong instructions to persist data. Poor experience for users in either way. And when distro defaults to "safe option", then the burden is on the sysadmin to toggle the switch but how is he supposed to decide when that is safe? First he has to understand what the problem actually is, then he has to audit all the applications using pmem whether they use the new instruction - which is IMO a lot of effort if you have a couple of applications and practically infeasible if you have more of them. So IMO the burden should be *on the application* to declare that it is aware of the new instructions to flush pmem on the platform and only to such application the kernel should give the trust to useMAP_SYNCmappings.
The "disaster" in my mind is this need to globally change the ABI for persistence semantics for all of Linux because one CPU wants a do over. What does a generic "MAP_SYNC_ENABLE" knob even mean to the existing deployed base of persistent memory applications? Yes, sysfs is awkward, but it's trying to provide some relief without imposing unexplainable semantics on everyone else. I think a comprehensive (overengineered) solution would involve not introducing another "I know what I'm doing" flag to the interface, but maybe requiring applications to call a pmem sync API in something like a vsyscall. Or, also overengineered, some binary translation / interpretation to actively detect and kill applications that deploy the old instructions. Something horrid like on first write fault to a MAP_SYNC try to look ahead in the binary for the correct sync sequence and kill the application otherwise. That would at least provide some enforcement and safety without requiring other architectures to consider what MAP_SYNC_ENABLE means to them.
Quoted 36 lines · Dan J
[ forgive formatting, a series of unfortunate events has me using Outlook for the moment ]
Quoted 17 lines
From: Jan Kara <[email protected]>
Quoted 10 lines
Quoted 7 lines
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
First I'd like to mention that hopefully the concern is mostly theoretical since as Aneesh wrote above, real persistent memory never shipped for PPC and so there are very few apps (if any) using the old way to ensure cache flushing.
Quoted 16 lines
But I'd like to understand why do you think per-process control is a recipe for disaster? Because from my POV the sysfs interface you propose is actually difficult to use in practice. As a distributor, you have hard time picking the default because you have a choice between picking safe option which is going to confuse users because of failing
MAP_SYNCand unsafe option where everyone will be happy until someone looses data because of some ancient application using wrong instructions to persist data. Poor experience for users in either way. And when distro defaults to "safe option", then the burden is on the sysadmin to toggle the switch but how is he supposed to decide when that is safe? First he has to understand what the problem actually is, then he has to audit all the applications using pmem whether they use the new instruction - which is IMO a lot of effort if you have a couple of applications and practically infeasible if you have more of them. So IMO the burden should be *on the application* to declare that it is aware of the new instructions to flush pmem on the platform and only to such application the kernel should give the trust to useMAP_SYNCmappings.
Quoted 15 lines
The "disaster" in my mind is this need to globally change the ABI for persistence semantics for all of Linux because one CPU wants a do over. What does a generic "
MAP_SYNC_ENABLE" knob even mean to the existing deployed base of persistent memory applications? Yes, sysfs is awkward, but it's trying to provide some relief without imposing unexplainable semantics on everyone else. I think a comprehensive (overengineered) solution would involve not introducing another "I know what I'm doing" flag to the interface, but maybe requiring applications to call a pmem sync API in something like a vsyscall. Or, also overengineered, some binary translation / interpretation to actively detect and kill applications that deploy the old instructions. Something horrid like on first write fault to aMAP_SYNCtry to look ahead in the binary for the correct sync sequence and kill the application otherwise. That would at least provide some enforcement and safety without requiring other architectures to consider whatMAP_SYNC_ENABLEmeans to them.
Thanks for explanation. So I absolutely agree that other architectures (and even older versions of POWER architecture) must not be influenced by the new tunable. That's why I wrote in my reply to Aneesh that I'd be for checking during mmap(2) with MAP_SYNC, whether we are in a situation where new PPC flush instructions are required and *only in that case* decide based on the prctl value whether MAP_SYNC should be allowed or not.
Whether this solution is overengineering or not depends on how you think it's likely there will be applications trying to use old flush instructions with MAP_SYNC on POWER10 platforms...
Honza
Signature · 5 lines
Jan Kara <[email protected]> SUSE Labs, CR _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 53 lines · Jan Kara
Quoted 36 lines · Dan J
[ forgive formatting, a series of unfortunate events has me using Outlook for the moment ]
Quoted 17 lines
From: Jan Kara <[email protected]>
Quoted 10 lines
Quoted 7 lines
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
First I'd like to mention that hopefully the concern is mostly theoretical since as Aneesh wrote above, real persistent memory never shipped for PPC and so there are very few apps (if any) using the old way to ensure cache flushing.
Quoted 16 lines
But I'd like to understand why do you think per-process control is a recipe for disaster? Because from my POV the sysfs interface you propose is actually difficult to use in practice. As a distributor, you have hard time picking the default because you have a choice between picking safe option which is going to confuse users because of failing
MAP_SYNCand unsafe option where everyone will be happy until someone looses data because of some ancient application using wrong instructions to persist data. Poor experience for users in either way. And when distro defaults to "safe option", then the burden is on the sysadmin to toggle the switch but how is he supposed to decide when that is safe? First he has to understand what the problem actually is, then he has to audit all the applications using pmem whether they use the new instruction - which is IMO a lot of effort if you have a couple of applications and practically infeasible if you have more of them. So IMO the burden should be *on the application* to declare that it is aware of the new instructions to flush pmem on the platform and only to such application the kernel should give the trust to useMAP_SYNCmappings.Quoted 15 lines
The "disaster" in my mind is this need to globally change the ABI for persistence semantics for all of Linux because one CPU wants a do over. What does a generic "
MAP_SYNC_ENABLE" knob even mean to the existing deployed base of persistent memory applications? Yes, sysfs is awkward, but it's trying to provide some relief without imposing unexplainable semantics on everyone else. I think a comprehensive (overengineered) solution would involve not introducing another "I know what I'm doing" flag to the interface, but maybe requiring applications to call a pmem sync API in something like a vsyscall. Or, also overengineered, some binary translation / interpretation to actively detect and kill applications that deploy the old instructions. Something horrid like on first write fault to aMAP_SYNCtry to look ahead in the binary for the correct sync sequence and kill the application otherwise. That would at least provide some enforcement and safety without requiring other architectures to consider whatMAP_SYNC_ENABLEmeans to them.
Quoted 7 lines
Thanks for explanation. So I absolutely agree that other architectures (and even older versions of POWER architecture) must not be influenced by the new tunable. That's why I wrote in my reply to Aneesh that I'd be for checking during mmap(2) with
MAP_SYNC, whether we are in a situation where new PPC flush instructions are required and *only in that case* decide based on the prctl value whetherMAP_SYNCshould be allowed or not.
v2 version of the patch series does that
https://lore.kernel.org/linuxppc-dev/[email protected]/
Quoted 4 lines
Whether this solution is overengineering or not depends on how you think it's likely there will be applications trying to use old flush instructions with
MAP_SYNCon POWER10 platforms...
Now considering that with ppc64 we never had a real persistent memory device available for the end-user to try and the new instructions are only needed on newer hardware, can we assume we have enough time to get the userspace to use new instructions?
As a safety net, we can keep the dax device-specific sysfs control. But in reality, by the time newer hardware gets released, we can get the distributions updated to flip the CONFIG_ARCH_MAP_SYNC_DISABLE=n?
With this:
1) vPMEM continues to work and since it is a volatile region. That doesn't need any flush instructions.
2) We get pmdk and other user applications updated to use new instructions and make sure updated packages are made available to all distributions
3) On newer hardware, the device will appear with a new compat string. Hence older distributions won't initialize pmem on newer hardware.
4) If we have a newer kernel with an older distro, we use the per namespace sysfs knob that prevents the usage of MAP_SYNC.
5) After a year or so we mark the CONFIG_ARCH_MAP_SYNC_DISABLE=n
on ppc64 when we are confident that everybody is using the new flush instruction.
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 91 lines · Aneesh Kumar K.V
Quoted 53 lines · Jan Kara
Quoted 36 lines · Dan J
[ forgive formatting, a series of unfortunate events has me using Outlook for the moment ]
Quoted 17 lines
From: Jan Kara <[email protected]>
Quoted 10 lines
Quoted 7 lines
Quoted 3 lines
These flags are device properties that affect the kernel and userspace's handling of persistence.
That will not handle the scenario with multiple applications using the same fsdax mount point where one is updated to use the new instruction and the other is not.
Right, it needs to be a global setting / flag day to switch from one regime to another. Per-process control is a recipe for disaster.
First I'd like to mention that hopefully the concern is mostly theoretical since as Aneesh wrote above, real persistent memory never shipped for PPC and so there are very few apps (if any) using the old way to ensure cache flushing.
Quoted 16 lines
But I'd like to understand why do you think per-process control is a recipe for disaster? Because from my POV the sysfs interface you propose is actually difficult to use in practice. As a distributor, you have hard time picking the default because you have a choice between picking safe option which is going to confuse users because of failing
MAP_SYNCand unsafe option where everyone will be happy until someone looses data because of some ancient application using wrong instructions to persist data. Poor experience for users in either way. And when distro defaults to "safe option", then the burden is on the sysadmin to toggle the switch but how is he supposed to decide when that is safe? First he has to understand what the problem actually is, then he has to audit all the applications using pmem whether they use the new instruction - which is IMO a lot of effort if you have a couple of applications and practically infeasible if you have more of them. So IMO the burden should be *on the application* to declare that it is aware of the new instructions to flush pmem on the platform and only to such application the kernel should give the trust to useMAP_SYNCmappings.Quoted 15 lines
The "disaster" in my mind is this need to globally change the ABI for persistence semantics for all of Linux because one CPU wants a do over. What does a generic "
MAP_SYNC_ENABLE" knob even mean to the existing deployed base of persistent memory applications? Yes, sysfs is awkward, but it's trying to provide some relief without imposing unexplainable semantics on everyone else. I think a comprehensive (overengineered) solution would involve not introducing another "I know what I'm doing" flag to the interface, but maybe requiring applications to call a pmem sync API in something like a vsyscall. Or, also overengineered, some binary translation / interpretation to actively detect and kill applications that deploy the old instructions. Something horrid like on first write fault to aMAP_SYNCtry to look ahead in the binary for the correct sync sequence and kill the application otherwise. That would at least provide some enforcement and safety without requiring other architectures to consider whatMAP_SYNC_ENABLEmeans to them.Quoted 7 lines
Thanks for explanation. So I absolutely agree that other architectures (and even older versions of POWER architecture) must not be influenced by the new tunable. That's why I wrote in my reply to Aneesh that I'd be for checking during mmap(2) with
MAP_SYNC, whether we are in a situation where new PPC flush instructions are required and *only in that case* decide based on the prctl value whetherMAP_SYNCshould be allowed or not.v2 version of the patch series does that
https://lore.kernel.org/linuxppc-dev/[email protected]/
Quoted 4 lines
Whether this solution is overengineering or not depends on how you think it's likely there will be applications trying to use old flush instructions with
MAP_SYNCon POWER10 platforms...Now considering that with ppc64 we never had a real persistent memory device available for the end-user to try and the new instructions are only needed on newer hardware, can we assume we have enough time to get the userspace to use new instructions?
As a safety net, we can keep the dax device-specific sysfs control. But in reality, by the time newer hardware gets released, we can get the distributions updated to flip the
CONFIG_ARCH_MAP_SYNC_DISABLE=n?With this:
1) vPMEM continues to work and since it is a volatile region. That doesn't need any flush instructions.2) We get pmdk and other user applications updated to use new instructions and make sure updated packages are made available to all distributions
3) On newer hardware, the device will appear with a new compat string. Hence older distributions won't initialize pmem on newer hardware.
Quoted 7 lines
4) If we have a newer kernel with an older distro, we use the per namespace sysfs knob that prevents the usage of
MAP_SYNC.5) After a year or so we mark the
CONFIG_ARCH_MAP_SYNC_DISABLE=non ppc64 when we are confident that everybody is using the new flush instruction.
Is this approach ok for distributions? If so I can repost the series dropping the prctl change.
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 971
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.
Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to use MAP_SYNC on these devices would need to use prctl(MMF_DISABLE_MAP_SYNC, 0) but then these applications need to be aware of new instructions so this isn't that much additional burden...
Quoted 9 lines
With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
This is a bit of a gamble... I don't have too much trust in certification / verification because only the "big players" may do powerfail testing throughout enough that they'd uncover these problems. So the question really is: How many apps are out there using MAP_SYNC on ppc64? Hopefully not many given the HW didn't ship yet as you wrote but I have no real clue. Similarly there's a question: How many app writers will read manual for older ppc64 architecture and write apps that won't work reliably on POWER10? Again, I have no idea.
So the prctl would be IMHO a nice safety belt but I'm not 100% certain it will be needed...
Honza
Signature · 5 lines
Jan Kara <[email protected]> SUSE Labs, CR _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 48 lines · Jan Kara
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 971
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.
Quoted 8 lines
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow
MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to useMAP_SYNCon these devices would need to useprctl(MMF_DISABLE_MAP_SYNC, 0)but then these applications need to be aware of new instructions so this isn't that much additional burden...
I am not sure application would want to add that much details/knowledge about a platform in their code. I was expecting application to do
#ifdef __ppc64__
prctl(MAP_SYNC_ENABLE, 1, 0, 0, 0));#endif
a = mmap(NULL, PAGE_SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED_VALIDATE | MAP_SYNC, fd, 0);
For that code all the complexity that we add w.r.t ibm,persistent-memory-v2 is not useful. Do you see a value in making all these device specific rather than a conditional on __ppc64__?
Quoted 20 lines
Quoted 10 lines
With that I am wondering should we even have this patch? Can we expect userspace get updated to use new instruction?.
With ppc64 we never had a real persistent memory device available for end user to try. The available persistent memory stack was using vPMEM which was presented as a volatile memory region for which there is no need to use any of the flush instructions. We could safely assume that as we get applications certified/verified for working with pmem device on ppc64, they would all be using the new instructions?
This is a bit of a gamble... I don't have too much trust in certification / verification because only the "big players" may do powerfail testing throughout enough that they'd uncover these problems. So the question really is: How many apps are out there using
MAP_SYNCon ppc64? Hopefully not many given the HW didn't ship yet as you wrote but I have no real clue. Similarly there's a question: How many app writers will read manual for older ppc64 architecture and write apps that won't work reliably on POWER10? Again, I have no idea.
Quoted 4 lines
So the prctl would be IMHO a nice safety belt but I'm not 100% certain it will be needed...
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 63 lines · Aneesh Kumar K.V
Quoted 48 lines · Jan Kara
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.Quoted 8 lines
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow
MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to useMAP_SYNCon these devices would need to useprctl(MMF_DISABLE_MAP_SYNC, 0)but then these applications need to be aware of new instructions so this isn't that much additional burden...I am not sure application would want to add that much details/knowledge about a platform in their code. I was expecting application to do
#ifdef __ppc64__ prctl(MAP_SYNC_ENABLE, 1, 0, 0, 0));
Quoted 8 lines
#endif
a = mmap(NULL,PAGE_SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED_VALIDATE|MAP_SYNC, fd, 0);For that code all the complexity that we add w.r.t ibm,persistent-memory-v2 is not useful. Do you see a value in making all these device specific rather than a conditional on
__ppc64__?
If the vpmem devices continue to work with the old instruction on POWER10 then it makes sense to make this per-device.
Also adding a message to kernel log in case the application does not do the prctl would be helful for people migrating old code to POWER10.
Thanks
Michal
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 69 lines · Michal Suchánek
Quoted 68 lines · Aneesh Kumar K.V
Quoted 48 lines · Jan Kara
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.Quoted 8 lines
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow
MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to useMAP_SYNCon these devices would need to useprctl(MMF_DISABLE_MAP_SYNC, 0)but then these applications need to be aware of new instructions so this isn't that much additional burden...I am not sure application would want to add that much details/knowledge about a platform in their code. I was expecting application to do
#ifdef __ppc64__ prctl(MAP_SYNC_ENABLE, 1, 0, 0, 0));#endif
a = mmap(NULL,PAGE_SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED_VALIDATE|MAP_SYNC, fd, 0);
Quoted 6 lines
Quoted 3 lines
For that code all the complexity that we add w.r.t ibm,persistent-memory-v2 is not useful. Do you see a value in making all these device specific rather than a conditional on
__ppc64__?If the vpmem devices continue to work with the old instruction on POWER10 then it makes sense to make this per-device.
vPMEM doesn't have write_cache and hence it is synchronous even without using any specific flush instruction. The question is do we want to have different programming steps when running on vPMEM vs a persistent PMEM device on ppc64.
I will work on the device specific ENABLE flag and then we can compare the kernel complexity against the added benefit.
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 77 lines · Aneesh Kumar K.V
Quoted 69 lines · Michal Suchánek
Quoted 68 lines · Aneesh Kumar K.V
Quoted 48 lines · Jan Kara
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.Quoted 8 lines
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow
MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to useMAP_SYNCon these devices would need to useprctl(MMF_DISABLE_MAP_SYNC, 0)but then these applications need to be aware of new instructions so this isn't that much additional burden...I am not sure application would want to add that much details/knowledge about a platform in their code. I was expecting application to do
#ifdef __ppc64__ prctl(MAP_SYNC_ENABLE, 1, 0, 0, 0));#endif
a = mmap(NULL,PAGE_SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED_VALIDATE|MAP_SYNC, fd, 0);Quoted 6 lines
Quoted 3 lines
For that code all the complexity that we add w.r.t ibm,persistent-memory-v2 is not useful. Do you see a value in making all these device specific rather than a conditional on
__ppc64__?If the vpmem devices continue to work with the old instruction on POWER10 then it makes sense to make this per-device.
Quoted 7 lines
vPMEM doesn't have
write_cacheand hence it is synchronous even without using any specific flush instruction. The question is do we want to have different programming steps when running on vPMEM vs a persistent PMEM device on ppc64.I will work on the device specific ENABLE flag and then we can compare the kernel complexity against the added benefit.
I have posted an RFC v2 [1] that implements a device-specific MAP_SYNC enable/disable feature. The Posted changes also add a dax flag suggested by Dan. With device-specific MAP_SYNC enable/disable, it was just a sysfs file export of the same flag.
1. https://lore.kernel.org/linuxppc-dev/[email protected]/
-aneesh
_______________________________________________
Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]
Quoted 63 lines · Aneesh Kumar K.V
Quoted 48 lines · Jan Kara
Quoted 41 lines · Aneesh Kumar K.V
Quoted 30 lines · Jan Kara
Quoted 1 line · Aneesh Kumar K.V
Thanks Michal. I also missed Jeff in this email thread.
And I think you'll also need some of the sched maintainers for the prctl bits...
Quoted 13 lines · Michal Suchánek
Quoted 12 lines
Adding Jan
Quoted 9 lines · Aneesh Kumar K.V
With POWER10, architecture is adding new pmem flush and sync instructions. The kernel should prevent the usage of
MAP_SYNCif applications are not using the new instructions on newer hardware.This patch adds a prctl option
MAP_SYNC_ENABLEthat can be used to enable the usage ofMAP_SYNC. The kernel config option is added to allow the user to control whetherMAP_SYNCshould be enabled by default or not.Signed-off-by Aneesh Kumar K.V <[email protected]>...
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
kernel/fork.c +4
-0@@ -963,6 +963,12 @@ __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);static unsigned long default_dump_filter =MMF_DUMP_FILTER_DEFAULT ; +#ifdef CONFIG_ARCH_MAP_SYNC_DISABLE +unsigned long default_map_sync_mask =MMF_DISABLE_MAP_SYNC_MASK ; +#else +unsigned long default_map_sync_mask =0 ;Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
kernel/fork.c __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock); · line 969
Quoted 2 lines
kernel/fork.c +2
-0+#endif +I'm not sure CONFIG is really the right approach here. For a distro that would basically mean to disable
MAP_SYNCfor all PPC kernels unless application explicitly uses the right prctl. Shouldn't we rather initializedefault_map_sync_maskon boot based on whether the CPU we run on requires new flush instructions or not? Otherwise the patch looks sensible.Quoted 5 lines
yes that is correct. We ideally want to deny
MAP_SYNConly w.r.t POWER10. But on a virtualized platform there is no easy way to detect that. We could ideally hook this into the nvdimm driver where we look at the new compat string ibm,persistent-memory-v2 and then disableMAP_SYNC
if we find a device with the specific value.Quoted 8 lines
Hum, couldn't we set some flag for nvdimm devices with "ibm,persistent-memory-v2" property and then check it during mmap(2) time and when the device has this propery and the mmap(2) caller doesn't have the prctl set, we'd disallow
MAP_SYNC? That should make things mostly seamless, shouldn't it? Only apps that want to useMAP_SYNCon these devices would need to useprctl(MMF_DISABLE_MAP_SYNC, 0)but then these applications need to be aware of new instructions so this isn't that much additional burden...I am not sure application would want to add that much details/knowledge about a platform in their code. I was expecting application to do
#ifdef __ppc64__ prctl(MAP_SYNC_ENABLE, 1, 0, 0, 0));
Quoted 8 lines
#endif
a = mmap(NULL,PAGE_SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED_VALIDATE|MAP_SYNC, fd, 0);For that code all the complexity that we add w.r.t ibm,persistent-memory-v2 is not useful. Do you see a value in making all these device specific rather than a conditional on
__ppc64__?
Yes, from the application POV the code would look like this plus the application would use instructions appropriate for POWER10 for flushing caches...
Honza
Signature · 5 lines
Jan Kara <[email protected]> SUSE Labs, CR _______________________________________________ Linux-nvdimm mailing list -- [email protected] To unsubscribe send an email to [email protected]