RFC: switch the remaining architectures to use generic GUP
Subsystems (6)
Hi Linus and maintainers,
below is a series to switch mips, sh and sparc64 to use the generic GUP code so that we only have one codebase to touch for further improvements to this code. I don't have hardware for any of these architectures, and generally no clue about their page table management, so handle with care. But it at least survives a basic defconfig compile test..
PATCH 1/6 MIPS: use the generic get_user_pages_fast code
The mips code is mostly equivalent to the generic one, minus various bugfixes and one and a half arch overrides that this patch adds to pgtable.h.
Note that this defines ARCH_HAS_PTE_SPECIAL for mips as mips has pte_special and pte_mkspecial implemented and used in the existing gup code. They are no-op stubs, though which makes me a little unsure if this is really right thing to do.
--- arch/mips/Kconfig | 2 + arch/mips/include/asm/pgtable.h | 30 ++++ arch/mips/mm/Makefile | 1 - arch/mips/mm/gup.c | 303 -------------------------------- 4 files changed, 32 insertions(+), 304 deletions(-) delete mode 100644 arch/mips/mm/gup.c
arch/mips/Kconfig +2 -0
@@ -6,6 +6,7 @@ config MIPS select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_ELF_RANDOMIZE + select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_SUPPORTS_UPROBES
@@ -55,6 +56,7 @@ config MIPS select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER + select HAVE_GENERIC_GUP select HAVE_IDE select HAVE_IOREMAP_PROT select HAVE_IRQ_EXIT_ON_IRQ_STACK
arch/mips/include/asm/pgtable.h +30 -0
@@ -20,6 +20,7 @@#include <asm/cmpxchg.h> #include <asm/io.h> #include <asm/pgtable-bits.h> +#include <asm/cpu-features.h> struct mm_struct ;struct vm_area_struct ;
@@ -651,4 +652,33 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,*/ #define pgtable_cache_init () do { } while (0) +static inline bool gup_fast_permitted (unsigned long start,int nr_pages) +{ +unsigned long len = (unsigned long )nr_pages <<PAGE_SHIFT ; +unsigned long end = start + len; + +return !cpu_has_dc_aliases && end >= start; +} +#define gup_fast_permitted gup_fast_permitted + +#if defined(CONFIG_PHYS_ADDR_T_64BIT ) && defined(CONFIG_CPU_MIPS32 ) +static inline pte_t gup_get_pte (pte_t *ptep) +{ +pte_t pte; + +retry: + pte.pte_low = ptep->pte_low; +smp_rmb (); + pte.pte_high = ptep->pte_high; +smp_rmb (); +if (unlikely (pte.pte_low != ptep->pte_low)) + goto retry; + +return pte; +} +#define gup_get_pte gup_get_pte +#endif + +static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, +#endif /* _ASM_PGTABLE_H */
arch/mips/mm/Makefile +0 -1
@@ -7,7 +7,6 @@ obj-y += cache.o obj-y += context.o obj-y += extable.o obj-y += fault.o -obj-y += gup.o obj-y += init.o obj-y += mmap.o obj-y += page.o
arch/mips/mm/gup.c deleted +0 -303
303 lines not on this page · raw message
Signature · 1 line
2.20.1
PATCH 2/6 sh: add a missing pud_page definition
In reply to Christoph Hellwigsh oddly enough had pud_page_vaddr, but not pud_page.
--- arch/sh/include/asm/pgtable-3level.h | 1 + 1 file changed, 1 insertion(+)
arch/sh/include/asm/pgtable-3level.h +1 -0
@@ -37,6 +37,7 @@ static inline unsigned long pud_page_vaddr(pud_t pud) {return pud_val (pud); } +#define pud_page (pud) virt_to_page((void *)pud_page_vaddr(pud))#define pmd_index (address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))static inline pmd_t *pmd_offset (pud_t *pud,unsigned long address)
Signature · 1 line
2.20.1
PATCH 3/6 sh: use the generic get_user_pages_fast code
In reply to Christoph HellwigThe sh code is mostly equivalent to the generic one, minus various bugfixes and two arch overrides that this patch adds to pgtable.h.
--- arch/sh/Kconfig | 1 + arch/sh/include/asm/pgtable.h | 85 +++++++++++ arch/sh/mm/Makefile | 2 +- arch/sh/mm/gup.c | 277 ---------------------------------- 4 files changed, 87 insertions(+), 278 deletions(-) delete mode 100644 arch/sh/mm/gup.c
arch/sh/Kconfig +1 -0
@@ -14,6 +14,7 @@ config SUPERH select HAVE_ARCH_TRACEHOOK select HAVE_PERF_EVENTS select HAVE_DEBUG_BUGVERBOSE + select HAVE_GENERIC_GUP select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAVE_NMI_SAFE_CMPXCHG if (GUSA_RB || CPU_SH4A) select ARCH_HAS_GCOV_PROFILE_ALL
arch/sh/include/asm/pgtable.h +85 -0
91 lines not on this page · raw message
arch/sh/mm/Makefile +1 -1
@@ -17,7 +17,7 @@ cacheops-$(CONFIG_CPU_SHX3) += cache-shx3.o obj-y += $(cacheops-y) mmu-y := nommu.o extable_32.o -mmu-$(CONFIG_MMU) := extable_$(BITS).o fault.o gup.o ioremap.o kmap.o \ +mmu-$(CONFIG_MMU) := extable_$(BITS).o fault.o ioremap.o kmap.o \ pgtable.o tlbex_$(BITS).o tlbflush_$(BITS).o obj-y += $(mmu-y)
arch/sh/mm/gup.c deleted +0 -277
277 lines not on this page · raw message
Signature · 1 line
2.20.1
PATCH 4/6 mm: add a gup_fixup_start_addr hook
In reply to Christoph HellwigThis will allow sparc64 to override its ADI tags for get_user_pages and get_user_pages_fast. I have no idea why this is not required for plain old get_user_pages, but it keeps the existing sparc64 behavior.
--- mm/gup.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
mm/gup.c +6 -2
@@ -2117,6 +2117,10 @@ static void gup_pgd_range(unsigned long addr, unsigned long end, }while (pgdp++, addr = next, addr != end); } +#ifndef gup_fixup_start_addr +#define gup_fixup_start_addr (start) (start) +#endif +#ifndef gup_fast_permitted/* * Check if it's allowed to use __get_user_pages_fast() for the range, or
@@ -2145,7 +2149,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,unsigned long flags;int nr =0 ; - start &=PAGE_MASK ; + start =gup_fixup_start_addr (start) &PAGE_MASK ; len = (unsigned long ) nr_pages <<PAGE_SHIFT ; end = start + len;
@@ -2218,7 +2222,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages,unsigned long addr, len, end;int nr =0 , ret =0 ; - start &=PAGE_MASK ; + start =gup_fixup_start_addr (start) &PAGE_MASK ; addr = start; len = (unsigned long ) nr_pages <<PAGE_SHIFT ; end = start + len;
Signature · 1 line
2.20.1
Re: [PATCH 4/6] mm: add a gup_fixup_start_addr hook
[ Adding Khalid, who added the sparc64 code ]
Quoted 5 lines · Christoph Hellwig
This will allow sparc64 to override its ADI tags for
get_user_pagesandget_user_pages_fast. I have no idea why this is not required for plain oldget_user_pages, but it keeps the existing sparc64 behavior.
This is actually generic. ARM64 has tagged pointers too. Right now the system call interfaces are all supposed to mask off the tags, but there's been noise about having the kernel understand them.
mm/gup.c static void gup_pgd_range(unsigned long addr, unsigned long end, · line 2122
Quoted 3 lines · That
mm/gup.c +3
-0+#ifndef gup_fixup_start_addr +#define gup_fixup_start_addr (start) (start) +#endif
I'd rather name this much more specifically (ie make it very much about "clean up pointer tags") and I'm also not clear on why sparc64 actually wants this. I thought the sparc64 rules were the same as the (current) arm64 rules: any addresses passed to the kernel have to be the non-tagged ones.
As you say, nothing *else* in the kernel does that address cleanup, why should get_user_pages_fast() do it?
David? Khalid? Why does sparc64 actually need this? It looks like the generic get_user_pages() doesn't do it.
Linus
Re: [PATCH 4/6] mm: add a gup_fixup_start_addr hook
Quoted 25 lines · Linus Torvalds
[ Adding Khalid, who added the sparc64 code ]
Quoted 5 lines · Christoph Hellwig
This will allow sparc64 to override its ADI tags for
get_user_pagesandget_user_pages_fast. I have no idea why this is not required for plain oldget_user_pages, but it keeps the existing sparc64 behavior.This is actually generic. ARM64 has tagged pointers too. Right now the system call interfaces are all supposed to mask off the tags, but there's been noise about having the kernel understand them.
mm/gup.c static void gup_pgd_range(unsigned long addr, unsigned long end, · line 2122
Quoted 3 lines · That
mm/gup.c +3
-0+#ifndef gup_fixup_start_addr +#define gup_fixup_start_addr (start) (start) +#endif I'd rather name this much more specifically (ie make it very much about "clean up pointer tags") and I'm also not clear on why sparc64 actually wants this. I thought the sparc64 rules were the same as the (current) arm64 rules: any addresses passed to the kernel have to be the non-tagged ones.
Quoted 6 lines
As you say, nothing *else* in the kernel does that address cleanup, why should
get_user_pages_fast()do it?David? Khalid? Why does sparc64 actually need this? It looks like the generic
get_user_pages()doesn't do it.
There is another discussion going on about tagged pointers on ARM64 and intersection with sparc64 code. I agree there is a generic need to mask off tags for kernel use now that ARM64 is also looking into supporting memory tagging. The need comes from sparc64 not storing tagged address in VMAs. It is not practical to store tagged addresses in VMAs because manipulation of address tags is done entirely in userspace on sparc64. Userspace is free to change tags on an address range at any time without involving kernel and constantly rotating tags is actually a security feature even. This makes it impractical for kernel to try to keep up with constantly changing tagged addresses in VMAs. Untagged addresses in VMAs means any find_vma() and brethren calls need to be passed an untagged address.
On sparc64, my intent was to support address tagging for dynamically allocated data buffers only (malloc, mmap and shm specifically) and not for any generic system calls which limited the scope and amount of untagging needed in the kernel. ARM64 is working to add transparent tagged address support at C library level. Adding tagged addresses to C library requires every possible call into kernel to either handle tagged addresses or untag address at some point. Andrey found out it is not as easy as untagging addresses in functions that search through vma. Callers of find_vma() and others tend to do address arithmetic on the address stored in vma that is returned. This requires a more complex solution than just stripping tags in vma lookup routines.
Since untagging addresses is a generic need required for far more than gup, I prefer the way Andrey wrote it -
<https://patchwork.kernel.org/patch/10923637/>
--
Khalid
Re: [PATCH 4/6] mm: add a gup_fixup_start_addr hook
Quoted 3 lines · Khalid Aziz
Since untagging addresses is a generic need required for far more than gup, I prefer the way Andrey wrote it -
<https://patchwork.kernel.org/patch/10923637/>
Linus, what do you think of picking up that trivial prep patch for 5.2? That way the arm64 and get_user_pages series can progress independently for 5.3.
Re: [PATCH 4/6] mm: add a gup_fixup_start_addr hook
In reply to Christoph HellwigHi Christoph,
mm/gup.c static void gup_pgd_range(unsigned long addr, unsigned long end, · line 2122
Quoted 11 lines · Christoph Hellwig
mm/gup.c +3
-0@@ -2117,6 +2117,10 @@ static void gup_pgd_range(unsigned long addr, unsigned long end, }while (pgdp++, addr = next, addr != end); } +#ifndef gup_fixup_start_addr +#define gup_fixup_start_addr (start) (start) +#endif
As you pointed out in a subsequent reply, we could use the untagged_addr() macro from Andrey (or a shorter "untag_addr" if you want it to look like a verb).
Quoted 13 lines
mm/gup.c +1
-1#ifndef gup_fast_permitted/* * Check if it's allowed to use __get_user_pages_fast() for the range, or @@ -2145,7 +2149,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,unsigned long flags;int nr =0 ; - start &=PAGE_MASK ; + start =gup_fixup_start_addr (start) &PAGE_MASK ; len = (unsigned long ) nr_pages <<PAGE_SHIFT ; end = start + len;
mm/gup.c int get_user_pages_fast(unsigned long start, int nr_pages, · line 2228
Quoted 8 lines
mm/gup.c +1
-1@@ -2218,7 +2222,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages,unsigned long addr, len, end;int nr =0 , ret =0 ; - start &=PAGE_MASK ; + start =gup_fixup_start_addr (start) &PAGE_MASK ; addr = start; len = (unsigned long ) nr_pages <<PAGE_SHIFT ; end = start + len;
In Andrey's patch [1] we don't fix __get_user_pages_fast(), only __get_user_pages() as it needs to do a find_vma() search. I wonder whether this is actually necessary for the *_fast() versions. If the top byte is non-zero (i.e. tagged address), 'end' would also have the same tag. The page table macros like pgd_index() and pgd_addr_end() already take care of masking out the top bits (at least for arm64) since they need to work on kernel address with the top bits all 1. So gup_pgd_range() should cope with tagged addresses already.
Signature · 1 line
Catalin
PATCH 5/6 sparc64: use the generic get_user_pages_fast code
In reply to Christoph HellwigThe sparc64 code is mostly equivalent to the generic one, minus various bugfixes and two arch overrides that this patch adds to pgtable.h.
--- arch/sparc/Kconfig | 1 + arch/sparc/include/asm/pgtable_64.h | 40 ++++ arch/sparc/mm/Makefile | 2 +- arch/sparc/mm/gup.c | 340 ---------------------------- 4 files changed, 42 insertions(+), 341 deletions(-) delete mode 100644 arch/sparc/mm/gup.c
arch/sparc/Kconfig +1 -0
@@ -28,6 +28,7 @@ config SPARC select RTC_DRV_M48T59 select RTC_SYSTOHC select HAVE_ARCH_JUMP_LABEL if SPARC64 + select HAVE_GENERIC_GUP if SPARC64 select GENERIC_IRQ_SHOW select ARCH_WANT_IPC_PARSE_VERSION select GENERIC_PCI_IOMAP
arch/sparc/include/asm/pgtable_64.h +40 -0
@@ -1075,6 +1075,46 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, }#define io_remap_pfn_range io_remap_pfn_range +static inline unsigned long gup_fixup_start_addr (unsigned long start) +{ +if (adi_capable ()) { +long addr = start; + +/* If userspace has passed a versioned address, kernel +* will not find it in the VMAs since it does not store +* the version tags in the list of VMAs. Storing version +* tags in list of VMAs is impractical since they can be +* changed any time from userspace without dropping into +* kernel. Any address search in VMAs will be done with +* non-versioned addresses. Ensure the ADI version bits +* are dropped here by sign extending the last bit before +* ADI bits. IOMMU does not implement version tags. +*/ +return (addr << (long )adi_nbits ()) >> (long )adi_nbits (); + } + +return start; +} +#define gup_fixup_start_addr gup_fixup_start_addr + +static inline bool pte_access_permitted (pte_t pte,bool write) +{ +u64 prot; + +if (tlb_type = hypervisor) { + prot = _PAGE_PRESENT_4V | _PAGE_P_4V; +if (prot) + prot |= _PAGE_WRITE_4V; + }else { + prot = _PAGE_PRESENT_4U | _PAGE_P_4U; +if (write) + prot |= _PAGE_WRITE_4U; + } + +return (pte_val (pte) & (prot | _PAGE_SPECIAL)) = prot; +} +#define pte_access_permitted pte_access_permitted +#include <asm/tlbflush.h> #include <asm-generic/pgtable.h>
arch/sparc/mm/Makefile +1 -1
@@ -5,7 +5,7 @@ asflags-y := -ansi ccflags-y := -Werror -obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o +obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o obj-y += fault_$(BITS).o obj-y += init_$(BITS).o obj-$(CONFIG_SPARC32) += extable.o srmmu.o iommu.o io-unit.o
arch/sparc/mm/gup.c deleted +0 -340
340 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [PATCH 5/6] sparc64: use the generic get_user_pages_fast code
From: Christoph Hellwig <[email protected]>
Date: Sat, 25 May 2019 15:32:02 +0200
Quoted 4 lines
The sparc64 code is mostly equivalent to the generic one, minus various bugfixes and two arch overrides that this patch adds to
pgtable.h.Signed-off-by Christoph Hellwig <[email protected]>
PATCH 6/6 mm: don't allow non-generic get_user_pages_fast implementations
In reply to Christoph HellwigAdd an explicit ifdef instead of the weak functions for the stubs so that we can't let new get_user_pages_fast implementation slip in.
--- mm/util.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
mm/util.c +6 -5
@@ -300,6 +300,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) }#endif +#ifndef CONFIG_HAVE_GENERIC_GUP /* * Like get_user_pages_fast() except its IRQ-safe in that it won't fall * back to the regular GUP.
@@ -308,8 +309,8 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)* If the architecture does not support this function, simply return with no * pages pinned. */ -int __weak __get_user_pages_fast (unsigned long start, -int nr_pages,int write,struct page **pages) +int __get_user_pages_fast (unsigned long start,int nr_pages,int write, +struct page **pages) {return 0 ; }
@@ -339,13 +340,13 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int __weak get_user_pages_fast (unsigned long start, -int nr_pages,unsigned int gup_flags, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {return get_user_pages_unlocked (start, nr_pages, pages, gup_flags); }EXPORT_SYMBOL_GPL (get_user_pages_fast); +#endif /* !CONFIG_HAVE_GENERIC_GUP */ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, unsigned long len, unsigned long prot,
Signature · 1 line
2.20.1
Looks good to me apart from the question about sparc64 (that you also raised) and requesting that interface to be re-named if it is really needed.
Let's just do it (but presumably for 5.3), and any architecture that doesn't react to this and gets broken because it wasn't tested can get fixed up later when/if they notice.
Linus
Quoted 7 lines · Linus Torvalds
Looks good to me apart from the question about sparc64 (that you also raised) and requesting that interface to be re-named if it is really needed.
Let's just do it (but presumably for 5.3), and any architecture that doesn't react to this and gets broken because it wasn't tested can get fixed up later when/if they notice.
FYI, my compile testing was very basic and a few issues showed up from the build bot later on. I'll keep the branch here uptodate for now:
http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/generic-gup
and won't resend until we make progress on the pointer tagging thing. I've also got a few follow on patches on top, so they might be ready by then as well.