various improvements to the GUP interface
Subsystems (8)
v2 cover various improvements to the GUP interface
A series of fixes to simplify and improve the GUP interface with an eye to providing groundwork to future improvements:-
* __access_remote_vm() and access_remote_vm() are functionally identical, so make the former static such that in future we can potentially change the external-facing implementation details of this function.
* Extend is_valid_gup_args() to cover the missing FOLL_TOUCH case, and simplify things by defining INTERNAL_GUP_FLAGS to check against.
* Adjust __get_user_pages_locked() to explicitly treat a failure to pin any pages as an error in all circumstances other than FOLL_NOWAIT being specified, bringing it in line with the nommu implementation of this function.
* (With many thanks to Arnd who suggested this in the first instance) Update get_user_page_vma_remote() to explicitly only return a page or an error, simplifying the interface and avoiding the questionable IS_ERR_OR_NULL() pattern.
v2:
* Corrected spelling mistakes in patches 1 and 2 (thanks PeterZ).
* Made things WARN_ON_ONCE() where appropriate and dropped unnecessary checks (thanks David).
v1:
https://lore.kernel.org/all/[email protected]/
Lorenzo Stoakes (4):
mm: make __access_remote_vm() static
mm/gup: explicitly define and check internal GUP flags, disallow
FOLL_TOUCH
mm/gup: make failure to pin an error if FOLL_NOWAIT not specified mm/gup: adapt get_user_page_vma_remote() to never return NULL
arch/arm64/kernel/mte.c | 4 ++-- include/linux/mm.h | 14 +++++++++----- kernel/events/uprobes.c | 4 ++-- kernel/ptrace.c | 2 +- mm/gup.c | 16 +++++++++++++--- mm/internal.h | 3 +++ mm/memory.c | 7 +++---- mm/nommu.c | 4 ++-- 8 files changed, 35 insertions(+), 19 deletions(-)
--
2.42.0
PATCH v2 1/4 mm: make __access_remote_vm() static
access_remote_vm() passes through parameters to __access_remote_vm() directly, so remove the __access_remote_vm() function from mm.h and use access_remote_vm() in the one caller that needs it (ptrace_access_vm()).
This allows future adjustments to the GUP-internal __access_remote_vm() function while keeping the access_remote_vm() function stable.
--- include/linux/mm.h | 2 -- kernel/ptrace.c | 2 +- mm/memory.c | 4 ++-- mm/nommu.c | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-)
include/linux/mm.h +0 -2
@@ -2415,8 +2415,6 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags);extern int access_remote_vm (struct mm_struct *mm,unsigned long addr,void *buf,int len,unsigned int gup_flags); -extern int __access_remote_vm (struct mm_struct *mm,unsigned long addr, -void *buf,int len,unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages,
kernel/ptrace.c +1 -1
@@ -59,7 +59,7 @@ int ptrace_access_vm(struct task_struct *tsk, unsigned long addr,return 0 ; } - ret =__access_remote_vm (mm, addr, buf, len, gup_flags); + ret =access_remote_vm (mm, addr, buf, len, gup_flags);mmput (mm);return ret;
mm/memory.c +2 -2
@@ -5881,8 +5881,8 @@ EXPORT_SYMBOL_GPL(generic_access_phys);/* * Access another process' address space as given in mm. */ -int __access_remote_vm (struct mm_struct *mm,unsigned long addr,void *buf, -int len,unsigned int gup_flags) +static int __access_remote_vm (struct mm_struct *mm,unsigned long addr, +void *buf,int len,unsigned int gup_flags) {void *old_buf = buf;int write = gup_flags &FOLL_WRITE ;
mm/nommu.c +2 -2
@@ -1651,8 +1651,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, }EXPORT_SYMBOL (filemap_map_pages); -int __access_remote_vm (struct mm_struct *mm,unsigned long addr,void *buf, -int len,unsigned int gup_flags) +static int __access_remote_vm (struct mm_struct *mm,unsigned long addr, +void *buf,int len,unsigned int gup_flags) {struct vm_area_struct *vma;int write = gup_flags &FOLL_WRITE ;
Signature · 1 line
2.42.0
PATCH v2 2/4 mm/gup: explicitly define and check internal GUP flags, disallow FOLL_TOUCH
In reply to Lorenzo StoakesRather than open-coding a list of internal GUP flags in is_valid_gup_args(), define which ones are internal.
In addition, explicitly check to see if the user passed in FOLL_TOUCH somehow, as this appears to have been accidentally excluded.
--- mm/gup.c | 5 ++--- mm/internal.h | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-)
mm/gup.c +2 -3
@@ -2227,12 +2227,11 @@ static bool is_valid_gup_args(struct page **pages, int *locked,/* * These flags not allowed to be specified externally to the gup * interfaces: -* - FOLL_PIN/FOLL_TRIED/FOLL_FAST_ONLY are internal only +* - FOLL_TOUCH/FOLL_PIN/FOLL_TRIED/FOLL_FAST_ONLY are internal only * - FOLL_REMOTE is internal only and used on follow_page() * - FOLL_UNLOCKABLE is internal only and used if locked is !NULL */ -if (WARN_ON_ONCE (gup_flags & (FOLL_PIN |FOLL_TRIED |FOLL_UNLOCKABLE | -FOLL_REMOTE |FOLL_FAST_ONLY ))) +if (WARN_ON_ONCE (gup_flags &INTERNAL_GUP_FLAGS ))return false; gup_flags |= to_set;
mm/internal.h +3 -0
@@ -1018,6 +1018,9 @@ enum { FOLL_UNLOCKABLE = 1 << 21, }; +#define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \ + FOLL_FAST_ONLY | FOLL_UNLOCKABLE) +/* * Indicates for which pages that are write-protected in the page table, * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the
Signature · 1 line
2.42.0
PATCH v2 3/4 mm/gup: make failure to pin an error if FOLL_NOWAIT not specified
In reply to Lorenzo StoakesThere really should be no circumstances under which a non-FOLL_NOWAIT GUP operation fails to return any pages, so make this an error and warn on it.
To catch the trivial case, simply exit early if nr_pages == 0.
This brings __get_user_pages_locked() in line with the behaviour of its nommu variant.
--- mm/gup.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
mm/gup.c +11 -0
@@ -1471,6 +1471,9 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm,long ret, pages_done;bool must_unlock = false; +if (!nr_pages) +return 0 ; +/* * The internal caller expects GUP to manage the lock internally and the * lock must be released when this returns.
@@ -1595,6 +1598,14 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm,mmap_read_unlock (mm); *locked =0 ; } + +/* +* Failing to pin anything implies something has gone wrong (except when +* FOLL_NOWAIT is specified). +*/ +if (WARN_ON_ONCE (pages_done ==0 && !(flags &FOLL_NOWAIT ))) +return -EFAULT ; +return pages_done; }
Signature · 1 line
2.42.0
Re: [PATCH v2 3/4] mm/gup: make failure to pin an error if FOLL_NOWAIT not specified
Quoted 11 lines · Lorenzo Stoakes
There really should be no circumstances under which a non-FOLL_NOWAIT GUP operation fails to return any pages, so make this an error and warn on it.
To catch the trivial case, simply exit early if
nr_pages== 0.This brings
__get_user_pages_locked()in line with the behaviour of its nommu variant.Reviewed-by Arnd Bergmann <[email protected]>Signed-off-by Lorenzo Stoakes <[email protected]>---
Signature · 3 lines
Cheers, David / dhildenb
PATCH v2 4/4 mm/gup: adapt get_user_page_vma_remote() to never return NULL
In reply to Lorenzo Stoakesget_user_pages_remote() will never return 0 except in the case of FOLL_NOWAIT being specified, which we explicitly disallow.
This simplifies error handling for the caller and avoids the awkwardness of dealing with both errors and failing to pin. Failing to pin here is an error.
--- arch/arm64/kernel/mte.c | 4 ++-- include/linux/mm.h | 12 +++++++++--- kernel/events/uprobes.c | 4 ++-- mm/memory.c | 3 +-- 4 files changed, 14 insertions(+), 9 deletions(-)
arch/arm64/kernel/mte.c +2 -2
@@ -411,8 +411,8 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,struct page *page =get_user_page_vma_remote (mm, addr, gup_flags, &vma); -if (IS_ERR_OR_NULL (page)) { - err = page ==NULL ? -EIO :PTR_ERR (page); +if (IS_ERR (page)) { + err =PTR_ERR (page);break ; }
include/linux/mm.h +9 -3
@@ -2425,6 +2425,9 @@ long pin_user_pages_remote(struct mm_struct *mm, unsigned int gup_flags, struct page **pages, int *locked); +/* +* Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. +*/ static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, unsigned long addr, int gup_flags,
@@ -2432,12 +2435,15 @@ static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, {struct page *page;struct vm_area_struct *vma; -int got =get_user_pages_remote (mm, addr,1 , gup_flags, &page,NULL ); +int got; + +if (WARN_ON_ONCE (unlikely (gup_flags &FOLL_NOWAIT ))) +return ERR_PTR (-EINVAL ); + + got =get_user_pages_remote (mm, addr,1 , gup_flags, &page,NULL );if (got <0 )return ERR_PTR (got); -if (got ==0 ) -return NULL ; vma =vma_lookup (mm, addr);if (WARN_ON_ONCE (!vma)) {
kernel/events/uprobes.c +2 -2
@@ -474,8 +474,8 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, gup_flags |=FOLL_SPLIT_PMD ;/* Read the page with vaddr into memory */ old_page =get_user_page_vma_remote (mm, vaddr, gup_flags, &vma); -if (IS_ERR_OR_NULL (old_page)) -return old_page ?PTR_ERR (old_page) :0 ; +if (IS_ERR (old_page)) +return PTR_ERR (old_page); ret =verify_opcode (old_page, vaddr, &opcode);if (ret <=0 )
mm/memory.c +1 -2
@@ -5905,7 +5905,7 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,struct page *page =get_user_page_vma_remote (mm, addr, gup_flags, &vma); -if (IS_ERR_OR_NULL (page)) { +if (IS_ERR (page)) {/* We might need to expand the stack to access it */ vma =vma_lookup (mm, addr);if (!vma) {
@@ -5919,7 +5919,6 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,continue ; } -/* * Check if this is a VM_IO | VM_PFNMAP VMA, which * we can access using slightly different code.
Signature · 1 line
2.42.0
Re: [PATCH v2 4/4] mm/gup: adapt get_user_page_vma_remote() to never return NULL
Quoted 7 lines · Lorenzo Stoakes
get_user_pages_remote()will never return 0 except in the case ofFOLL_NOWAITbeing specified, which we explicitly disallow.This simplifies error handling for the caller and avoids the awkwardness of dealing with both errors and failing to pin. Failing to pin here is an error.
Quoted 5 lines
Suggested-by Arnd Bergmann <[email protected]>Reviewed-by Arnd Bergmann <[email protected]>Acked-by Catalin Marinas <[email protected]>Signed-off-by Lorenzo Stoakes <[email protected]>---
Signature · 3 lines
Cheers, David / dhildenb