Fix copy_from_user_nofault()
Subsystems (6)
cover Fix copy_from_user_nofault()
The original patch got submitted by Alexei Starovoitov with [0] and fixes issues that got also reported in [1].
This resubmission adds !pagefault_disabled() to the check in check_heap_object().
[0] https://lore.kernel.org/all/[email protected]/
[1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1033398
Alexei Starovoitov (2):
mm: Fix copy_from_user_nofault().
perf: Fix arch_perf_out_copy_user().
arch/x86/include/asm/perf_event.h | 2 -- arch/x86/lib/Makefile | 2 +- arch/x86/lib/usercopy.c | 55 ------------------------------- kernel/events/internal.h | 16 +-------- mm/maccess.c | 54 +++++++++++++++++++++++++----- mm/usercopy.c | 2 +- 6 files changed, 49 insertions(+), 82 deletions(-) delete mode 100644 arch/x86/lib/usercopy.c
Signature · 1 line
2.39.2
PATCH 1/2 mm: Fix copy_from_user_nofault().
From: Alexei Starovoitov <[email protected]>
There are several issues with copy_from_user_nofault():
- access_ok() is designed for user context only and for that reason it has WARN_ON_IN_IRQ() which triggers when bpf, kprobe, eprobe and perf on ppc are calling it from irq.
- it's missing nmi_uaccess_okay() which is a nop on all architectures except x86 where it's required.
The comment in arch/x86/mm/tlb.c explains the details why it's necessary. Calling copy_from_user_nofault() from bpf, [ke]probe without this check is not safe.
- __copy_from_user_inatomic() under CONFIG_HARDENED_USERCOPY is calling check_object_size()->__check_object_size()->check_heap_object()->find_vmap_area()->spin_lock() which is not safe to do from bpf, [ke]probe and perf due to potential deadlock.
Fix all three issues. At the end the copy_from_user_nofault() becomes equivalent to copy_from_user_nmi() from safety point of view with a difference in the return value.
--- mm/maccess.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-)
mm/maccess.c +13 -5
@@ -5,6 +5,7 @@#include <linux/export.h> #include <linux/mm.h> #include <linux/uaccess.h> +#include <asm/tlb.h> bool __weak copy_from_kernel_nofault_allowed (const void *unsafe_src,size_t size)
@@ -113,11 +114,18 @@ long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)long copy_from_user_nofault (void *dst,const void __user *src,size_t size) {long ret = -EFAULT ; -if (access_ok (src, size)) { -pagefault_disable (); - ret =__copy_from_user_inatomic (dst, src, size); -pagefault_enable (); - } + +if (!__access_ok (src, size)) +return ret; + +if (!nmi_uaccess_okay ()) +return ret; + +pagefault_disable (); +instrument_copy_from_user_before (dst, src, size); + ret =raw_copy_from_user (dst, src, size); +instrument_copy_from_user_after (dst, src, size, ret); +pagefault_enable ();if (ret)return -EFAULT ;
Signature · 1 line
2.39.2
PATCH 2/2 perf: Fix arch_perf_out_copy_user().
In reply to Florian LehnerFrom: Alexei Starovoitov <[email protected]>
There are several issues with arch_perf_out_copy_user(). On x86 it's the same as copy_from_user_nmi() and all is good, but on other archs:
- __access_ok() is missing.
Only on m68k, s390, parisc, sparc64 archs this function returns 'true'. Other archs must call it before user memory access.
- nmi_uaccess_okay() is missing.
- __copy_from_user_inatomic() issues under CONFIG_HARDENED_USERCOPY.
The latter two issues existed in copy_from_user_nofault() as well and were fixed in the previous patch.
This patch copies comments from copy_from_user_nmi() into mm/maccess.c and splits copy_from_user_nofault() into copy_from_user_nmi() that returns number of not copied bytes and copy_from_user_nofault() that returns -EFAULT or zero.
With that copy_from_user_nmi() becomes generic and is used by perf on all architectures.
--- arch/x86/include/asm/perf_event.h | 2 -- arch/x86/lib/Makefile | 2 +- arch/x86/lib/usercopy.c | 55 ------------------------------- kernel/events/internal.h | 16 +-------- mm/maccess.c | 48 ++++++++++++++++++++++----- mm/usercopy.c | 2 +- 6 files changed, 42 insertions(+), 83 deletions(-) delete mode 100644 arch/x86/lib/usercopy.c
arch/x86/include/asm/perf_event.h +0 -2
@@ -598,6 +598,4 @@ static __always_inline void perf_lopwr_cb(bool lopwr_in)static inline void amd_pmu_disable_virt (void ) { }#endif -#define arch_perf_out_copy_user copy_from_user_nmi -#endif /* _ASM_X86_PERF_EVENT_H */
arch/x86/lib/Makefile +1 -1
@@ -42,7 +42,7 @@ clean-files := inat-tables.c obj-$(CONFIG_SMP) += msr-smp.o cache-smp.o lib-y := delay.o misc.o cmdline.o cpu.o -lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o +lib-y += usercopy_$(BITS).o getuser.o putuser.o lib-y += memcpy_$(BITS).o lib-y += pc-conf-reg.o lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc.o copy_mc_64.o
arch/x86/lib/usercopy.c deleted +0 -55
55 lines not on this page · raw message
kernel/events/internal.h +1 -15
@@ -190,21 +190,7 @@ memcpy_skip(void *dst, const void *src, unsigned long n) DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip) -#ifndef arch_perf_out_copy_user -#define arch_perf_out_copy_user arch_perf_out_copy_user - -static inline unsigned long -arch_perf_out_copy_user (void *dst,const void *src,unsigned long n) -{ -unsigned long ret; - -pagefault_disable (); - ret =__copy_from_user_inatomic (dst, src, n); -pagefault_enable (); - -return ret; -} -#endif +#define arch_perf_out_copy_user copy_from_user_nmiDEFINE_OUTPUT_COPY (__output_copy_user, arch_perf_out_copy_user)
mm/maccess.c +39 -9
69 lines not on this page · raw message
mm/usercopy.c +1 -1
@@ -173,7 +173,7 @@ static inline void check_heap_object(const void *ptr, unsigned long n,return ; } -if (is_vmalloc_addr (ptr)) { +if (is_vmalloc_addr (ptr) && !pagefault_disabled ()) {struct vmap_area *area =find_vmap_area (addr);if (!area)
Signature · 1 line
2.39.2
Re: [RESUBMIT bpf-next 2/2] perf: Fix arch_perf_out_copy_user().
Quoted 225 lines · Florian Lehner
From: Alexei Starovoitov <[email protected]>
There are several issues with
arch_perf_out_copy_user(). On x86 it's the same ascopy_from_user_nmi()and all is good, but on other archs:-
__access_ok()is missing.
Only on m68k, s390, parisc, sparc64 archs this function returns 'true'. Other archs must call it before user memory access.
-nmi_uaccess_okay()is missing.
-__copy_from_user_inatomic()issues underCONFIG_HARDENED_USERCOPY.The latter two issues existed in
copy_from_user_nofault()as well and were fixed in the previous patch.This patch copies comments from
copy_from_user_nmi()intomm/maccess.cand splitscopy_from_user_nofault()intocopy_from_user_nmi()that returns number of not copied bytes andcopy_from_user_nofault()that returns-EFAULTor zero.
With thatcopy_from_user_nmi()becomes generic and is used by perf on all architectures.Signed-off-by Alexei Starovoitov <[email protected]>--- arch/x86/include/asm/perf_event.h | 2 -- arch/x86/lib/Makefile | 2 +- arch/x86/lib/usercopy.c | 55 ------------------------------- kernel/events/internal.h | 16 +-------- mm/maccess.c | 48 ++++++++++++++++++++++----- mm/usercopy.c | 2 +- 6 files changed, 42 insertions(+), 83 deletions(-) delete mode 100644 arch/x86/lib/usercopy.carch/x86/include/asm/perf_event.h +0
-2@@ -598,6 +598,4 @@ static __always_inline void perf_lopwr_cb(bool lopwr_in)static inline void amd_pmu_disable_virt (void ) { }#endif -#define arch_perf_out_copy_user copy_from_user_nmi -#endif /* _ASM_X86_PERF_EVENT_H */ arch/x86/lib/Makefile +1
-1@@ -42,7 +42,7 @@ clean-files := inat-tables.c obj-$(CONFIG_SMP) += msr-smp.o cache-smp.o lib-y := delay.o misc.o cmdline.o cpu.o -lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o +lib-y += usercopy_$(BITS).o getuser.o putuser.o lib-y += memcpy_$(BITS).o lib-y += pc-conf-reg.o lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc.o copy_mc_64.oarch/x86/lib/usercopy.c deleted +0
-55@@ -1,55 +0,0 @@ -/* -* User address space access functions. -* -* For licencing details see kernel-base/COPYING -*/ - -#include <linux/uaccess.h> -#include <linux/export.h> -#include <linux/instrumented.h> - -#include <asm/tlbflush.h> - -/** -* copy_from_user_nmi - NMI safe copy from user -* @to: Pointer to the destination buffer -* @from: Pointer to a user space address of the current task -* @n: Number of bytes to copy -* -* Returns: The number of not copied bytes. 0 is success, i.e. all bytes copied -* -* Contrary to other copy_from_user() variants this function can be called -* from NMI context. Despite the name it is not restricted to be called -* from NMI context. It is safe to be called from any other context as -* well. It disables pagefaults across the copy which means a fault will -* abort the copy. -* -* For NMI context invocations this relies on the nested NMI work to allow -* atomic faults from the NMI path; the nested NMI paths are careful to -* preserve CR2. -*/ -unsigned long -copy_from_user_nmi (void *to,const void __user *from,unsigned long n) -{ -unsigned long ret; - -if (!__access_ok (from, n)) -return n; - -if (!nmi_uaccess_okay ()) -return n; - -/* -* Even though this function is typically called from NMI/IRQ context -* disable pagefaults so that its behaviour is consistent even when -* called from other contexts. -*/ -pagefault_disable (); -instrument_copy_from_user_before (to, from, n); - ret =raw_copy_from_user (to, from, n); -instrument_copy_from_user_after (to, from, n, ret); -pagefault_enable (); - -return ret; -} -EXPORT_SYMBOL_GPL (copy_from_user_nmi);kernel/events/internal.h +1
-15@@ -190,21 +190,7 @@ memcpy_skip(void *dst, const void *src, unsigned long n) DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip) -#ifndef arch_perf_out_copy_user -#define arch_perf_out_copy_user arch_perf_out_copy_user - -static inline unsigned long -arch_perf_out_copy_user (void *dst,const void *src,unsigned long n) -{ -unsigned long ret; - -pagefault_disable (); - ret =__copy_from_user_inatomic (dst, src, n); -pagefault_enable (); - -return ret; -} -#endif +#define arch_perf_out_copy_user copy_from_user_nmiDEFINE_OUTPUT_COPY (__output_copy_user, arch_perf_out_copy_user)mm/maccess.c +39
-9@@ -103,17 +103,27 @@ long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) }/** -* copy_from_user_nofault(): safely attempt to read from a user-space location -* @dst: pointer to the buffer that shall take the data -* @src: address to read from. This must be a user address. -* @size: size of the data chunk +* copy_from_user_nmi - NMI safe copy from user +* @dst: Pointer to the destination buffer +* @src: Pointer to a user space address of the current task +* @size: Number of bytes to copy * -* Safely read from user address @src to the buffer at @dst. If a kernel fault -* happens, handle that and return -EFAULT. +* Returns: The number of not copied bytes. 0 is success, i.e. all bytes copied +* +* Contrary to other copy_from_user() variants this function can be called +* from NMI context. Despite the name it is not restricted to be called +* from NMI context. It is safe to be called from any other context as +* well. It disables pagefaults across the copy which means a fault will +* abort the copy. +* +* For NMI context invocations this relies on the nested NMI work to allow +* atomic faults from the NMI path; the nested NMI paths are careful to +* preserve CR2 on X86 architecture. */ -long copy_from_user_nofault (void *dst,const void __user *src,size_t size) +unsigned long +copy_from_user_nmi (void *dst,const void __user *src,unsigned long size) { -long ret = -EFAULT ; +unsigned long ret = size;if (!__access_ok (src, size))return ret;@@ -121,13 +131,33 @@ long copy_from_user_nofault(void *dst, const void __user *src, size_t size)if (!nmi_uaccess_okay ())return ret; +/* +* Even though this function is typically called from NMI/IRQ context +* disable pagefaults so that its behaviour is consistent even when +* called from other contexts. +*/ pagefault_disable ();instrument_copy_from_user_before (dst, src, size); ret =raw_copy_from_user (dst, src, size);instrument_copy_from_user_after (dst, src, size, ret);pagefault_enable (); -if (ret) +return ret; +} +EXPORT_SYMBOL_GPL (copy_from_user_nmi); + +/** +* copy_from_user_nofault(): safely attempt to read from a user-space location +* @dst: pointer to the buffer that shall take the data +* @src: address to read from. This must be a user address. +* @size: size of the data chunk +* +* Safely read from user address @src to the buffer at @dst. If a kernel fault +* happens, handle that and return -EFAULT. +*/ +long copy_from_user_nofault (void *dst,const void __user *src,size_t size) +{ +if (copy_from_user_nmi (dst, src, size))return -EFAULT ;return 0 ; }mm/usercopy.c +0
-0
mm/usercopy.c static inline void check_heap_object(const void *ptr, unsigned long n, · line 176
Quoted 5 lines
mm/usercopy.c +1
-1@@ -173,7 +173,7 @@ static inline void check_heap_object(const void *ptr, unsigned long n,return ; } -if (is_vmalloc_addr (ptr)) { +if (is_vmalloc_addr (ptr) && !pagefault_disabled ()) {
Florian,
thank you for taking over the patches.
This bit isn't right though.
This hunk needs to be in patch 1.
Then instead of open coding __copy_from_user_inatomic without check_object_size() it would be fine to only add __access_ok and nmi_uaccess_okay()
to copy_from_user_nofault() and keep __copy_from_user_inatomic().
The patch 2 can still remove copy_from_user_nmi() (adjusting return value, of course), since check_heap_object() will no longer dead lock due to !pagefault_disabled() in the patch 1.
Does this make sense?