Add gup fast + longterm and use it in HFI1
Subsystems (42)
cover Add gup fast + longterm and use it in HFI1
From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
HFI1 uses get_user_pages_fast() due to it performance advantages. Like RDMA, HFI1 pages can be held for a significant time. But get_user_pages_fast() does not protect against mapping of FS DAX pages.
Introduce a get_user_pages_fast_longterm() which retains the performance while also adding the FS DAX checks. XDP has also shown interest in using this functionality.[2]
[1] https://lkml.org/lkml/2019/2/11/237
[2] https://lkml.org/lkml/2019/2/11/1789
Ira Weiny (3):
mm/gup: Change "write" parameter to flags mm/gup: Introduce get_user_pages_fast_longterm() IB/HFI1: Use new get_user_pages_fast_longterm()
drivers/infiniband/hw/hfi1/user_pages.c | 2 +- include/linux/mm.h | 8 ++ mm/gup.c | 152 ++++++++++++++++-------- 3 files changed, 114 insertions(+), 48 deletions(-)
Signature · 1 line
2.20.1
PATCH 1/3 mm/gup: Change "write" parameter to flags
From: Ira Weiny <[email protected]>
In order to support more options in the GUP fast walk, change the write parameter to flags throughout the call stack.
This patch does not change functionality and passes FOLL_WRITE where write was previously used.
--- mm/gup.c | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-)
mm/gup.c +26 -26
@@ -1395,7 +1395,7 @@ static void undo_dev_pagemap(int *nr, int nr_start, struct page **pages)#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {struct dev_pagemap *pgmap =NULL ;int nr_start = *nr, ret =0 ;
@@ -1413,7 +1413,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,if (pte_protnone (pte)) goto pte_unmap; -if (!pte_access_permitted (pte, write)) +if (!pte_access_permitted (pte, flags &FOLL_WRITE )) goto pte_unmap;if (pte_devmap (pte)) {
@@ -1465,7 +1465,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,* useful to have gup_huge_pmd even if we can't operate on ptes. */ static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {return 0 ; }
@@ -1548,12 +1548,12 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,#endif static int gup_huge_pmd (pmd_t orig,pmd_t *pmdp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pmd_access_permitted (orig, write)) +if (!pmd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pmd_devmap (orig))
@@ -1586,12 +1586,12 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, }static int gup_huge_pud (pud_t orig,pud_t *pudp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pud_access_permitted (orig, write)) +if (!pud_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pud_devmap (orig))
@@ -1624,13 +1624,13 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, }static int gup_huge_pgd (pgd_t orig,pgd_t *pgdp,unsigned long addr, -unsigned long end,int write, +unsigned long end,unsigned int flags,struct page **pages,int *nr) {int refs;struct page *head, *page; -if (!pgd_access_permitted (orig, write)) +if (!pgd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;BUILD_BUG_ON (pgd_devmap (orig));
@@ -1661,7 +1661,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, }static int gup_pmd_range (pud_t pud,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pmd_t *pmdp;
@@ -1683,7 +1683,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,if (pmd_protnone (pmd))return 0 ; -if (!gup_huge_pmd (pmd, pmdp, addr, next, write, +if (!gup_huge_pmd (pmd, pmdp, addr, next, flags, pages, nr))return 0 ;
@@ -1693,9 +1693,9 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,* pmd format and THP pmd format */ if (!gup_huge_pd (__hugepd (pmd_val (pmd)), addr, -PMD_SHIFT , next, write, pages, nr)) +PMD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pte_range (pmd, addr, next, write, pages, nr)) + }else if (!gup_pte_range (pmd, addr, next, flags, pages, nr))return 0 ; }while (pmdp++, addr = next, addr != end);
@@ -1703,7 +1703,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, }static int gup_pud_range (p4d_t p4d,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pud_t *pudp;
@@ -1716,14 +1716,14 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,if (pud_none (pud))return 0 ;if (unlikely (pud_huge (pud))) { -if (!gup_huge_pud (pud, pudp, addr, next, write, +if (!gup_huge_pud (pud, pudp, addr, next, flags, pages, nr))return 0 ; }else if (unlikely (is_hugepd (__hugepd (pud_val (pud))))) {if (!gup_huge_pd (__hugepd (pud_val (pud)), addr, -PUD_SHIFT , next, write, pages, nr)) +PUD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pmd_range (pud, addr, next, write, pages, nr)) + }else if (!gup_pmd_range (pud, addr, next, flags, pages, nr))return 0 ; }while (pudp++, addr = next, addr != end);
@@ -1731,7 +1731,7 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end, }static int gup_p4d_range (pgd_t pgd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;p4d_t *p4dp;
@@ -1746,9 +1746,9 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,BUILD_BUG_ON (p4d_huge (p4d));if (unlikely (is_hugepd (__hugepd (p4d_val (p4d))))) {if (!gup_huge_pd (__hugepd (p4d_val (p4d)), addr, -P4D_SHIFT , next, write, pages, nr)) +P4D_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pud_range (p4d, addr, next, write, pages, nr)) + }else if (!gup_pud_range (p4d, addr, next, flags, pages, nr))return 0 ; }while (p4dp++, addr = next, addr != end);
@@ -1756,7 +1756,7 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end, }static void gup_pgd_range (unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pgd_t *pgdp;
@@ -1769,14 +1769,14 @@ static void gup_pgd_range(unsigned long addr, unsigned long end,if (pgd_none (pgd))return ;if (unlikely (pgd_huge (pgd))) { -if (!gup_huge_pgd (pgd, pgdp, addr, next, write, +if (!gup_huge_pgd (pgd, pgdp, addr, next, flags, pages, nr))return ; }else if (unlikely (is_hugepd (__hugepd (pgd_val (pgd))))) {if (!gup_huge_pd (__hugepd (pgd_val (pgd)), addr, -PGDIR_SHIFT , next, write, pages, nr)) +PGDIR_SHIFT , next, flags, pages, nr))return ; - }else if (!gup_p4d_range (pgd, addr, next, write, pages, nr)) + }else if (!gup_p4d_range (pgd, addr, next, flags, pages, nr))return ; }while (pgdp++, addr = next, addr != end); }
@@ -1830,7 +1830,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_save (flags); -gup_pgd_range (start, end, write, pages, &nr); +gup_pgd_range (start, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_restore (flags); }
@@ -1872,7 +1872,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_disable (); -gup_pgd_range (addr, end, write, pages, &nr); +gup_pgd_range (addr, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_enable (); ret = nr; }
Signature · 1 line
2.20.1
PATCH 2/3 mm/gup: Introduce get_user_pages_fast_longterm()
In reply to [email protected]From: Ira Weiny <[email protected]>
Users of get_user_pages_fast are not protected against mapping pages within FS DAX. Introduce a call which protects them.
We do this by checking for DEVMAP pages during the fast walk and falling back to the longterm gup call to check for FS DAX if needed.
--- include/linux/mm.h | 8 ++++ mm/gup.c | 102 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 22 deletions(-)
include/linux/mm.h +8 -0
@@ -1540,6 +1540,8 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,long get_user_pages_longterm (unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page **pages,struct vm_area_struct **vmas); +int get_user_pages_fast_longterm (unsigned long start,int nr_pages,bool write, +struct page **pages);#else static inline long get_user_pages_longterm(unsigned long start, unsigned long nr_pages, unsigned int gup_flags,
@@ -1547,6 +1549,11 @@ static inline long get_user_pages_longterm(unsigned long start, {return get_user_pages (start, nr_pages, gup_flags, pages, vmas); } +static inline int get_user_pages_fast_longterm (unsigned long start,int nr_pages, +bool write,struct page **pages) +{ +return get_user_pages_fast (start, nr_pages, write, pages); +}#endif /* CONFIG_FS_DAX */ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
@@ -2615,6 +2622,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */ static inline int vm_fault_to_errno (vm_fault_t vm_fault,int foll_flags) {
mm/gup.c +80 -22
143 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 38 lines · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Users of
get_user_pages_fastare not protected against mapping pages within FS DAX. Introduce a call which protects them.We do this by checking for DEVMAP pages during the fast walk and falling back to the longterm gup call to check for FS DAX if needed.
Signed-off-by Ira Weiny <[email protected]>include/linux/mm.h | 8 ++++ mm/gup.c | 102 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 22 deletions(-)include/linux/mm.h +8
-0+++ b/include/linux/mm.h@@ -1540,6 +1540,8 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,long get_user_pages_longterm (unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page **pages,struct vm_area_struct **vmas); +int get_user_pages_fast_longterm (unsigned long start,int nr_pages,bool write, +struct page **pages);#else static inline long get_user_pages_longterm(unsigned long start, unsigned long nr_pages, unsigned int gup_flags,@@ -1547,6 +1549,11 @@ static inline long get_user_pages_longterm(unsigned long start, {return get_user_pages (start, nr_pages, gup_flags, pages, vmas); } +static inline int get_user_pages_fast_longterm (unsigned long start,int nr_pages, +bool write,struct page **pages) +{ +return get_user_pages_fast (start, nr_pages, write, pages); +}#endif /* CONFIG_FS_DAX */ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
include/linux/mm.h struct page *follow_page(struct vm_area_struct *vma, unsigned long address, · line 2625
Quoted 4 lines
include/linux/mm.h +1
-0@@ -2615,6 +2622,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */
If we are adding a new flag, maybe we should get rid of the 'longterm' entry points and just rely on the callers to pass the flag?
Jason
John Hubbard
reviewerRe: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
[...]
Quoted 17 lines
include/linux/mm.h struct page *follow_page(struct vm_area_struct *vma, unsigned long address, · line 2625
Quoted 13 lines
include/linux/mm.h +6
-0+static inline int get_user_pages_fast_longterm (unsigned long start,int nr_pages, +bool write,struct page **pages) +{ +return get_user_pages_fast (start, nr_pages, write, pages); +}#endif /* CONFIG_FS_DAX */ int get_user_pages_fast(unsigned long start, int nr_pages, int write,@@ -2615,6 +2622,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */ If we are adding a new flag, maybe we should get rid of the 'longterm' entry points and just rely on the callers to pass the flag?
Quoted 2 lines
Jason
+1, I agree that the overall get_user_pages*() API family will be cleaner *without* get_user_pages_longterm*() calls. And this new flag makes that possible. So I'd like to see the "longerm" call replaced with just passing this flag. Maybe even as part of this patchset, but either way.
Taking a moment to reflect on where I think this might go eventually (the notes below do not need to affect your patchset here, but this seems like a good place to mention this):
It seems to me that the longterm vs. short-term is of questionable value. It's actually better to just call get_user_pages(), and then if it really is long-term enough to matter internally, we'll see the pages marked as gup-pinned. If the gup pages are released before anyone (filesystem, that is) notices, then it must have been short term.
Doing it that way is self-maintaining. Of course, this assumes that we end up with a design that doesn't require being told, by the call sites, that a given gup call is intended for "long term" use. So I could be wrong about this direction, but let's please consider the possibility.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 21 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
[...]
Quoted 17 lines
include/linux/mm.h struct page *follow_page(struct vm_area_struct *vma, unsigned long address, · line 2625
Quoted 13 lines
include/linux/mm.h +6
-0+static inline int get_user_pages_fast_longterm (unsigned long start,int nr_pages, +bool write,struct page **pages) +{ +return get_user_pages_fast (start, nr_pages, write, pages); +}#endif /* CONFIG_FS_DAX */ int get_user_pages_fast(unsigned long start, int nr_pages, int write,@@ -2615,6 +2622,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */ If we are adding a new flag, maybe we should get rid of the 'longterm' entry points and just rely on the callers to pass the flag?
Quoted 7 lines
Quoted 2 lines
Jason
+1, I agree that the overall get_user_pages*() API family will be cleaner *without* get_user_pages_longterm*() calls. And this new flag makes that possible. So I'd like to see the "longerm" call replaced with just passing this flag. Maybe even as part of this patchset, but either way.
Yes I've thought about this as well. I have a couple of different versions of this series which I've been mulling over and this was one of the other variations. But see below...
Quoted 6 lines
Taking a moment to reflect on where I think this might go eventually (the notes below do not need to affect your patchset here, but this seems like a good place to mention this):
It seems to me that the longterm vs. short-term is of questionable value.
This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing FOLL_LONGTERM seemed like the cleanest way to do that even if we will not need the distinction in the future... :-(
Quoted 9 lines
It's actually better to just call
get_user_pages(), and then if it really is long-term enough to matter internally, we'll see the pages marked as gup-pinned. If the gup pages are released before anyone (filesystem, that is) notices, then it must have been short term.Doing it that way is self-maintaining. Of course, this assumes that we end up with a design that doesn't require being told, by the call sites, that a given gup call is intended for "long term" use. So I could be wrong about this direction, but let's please consider the possibility.
This is why I've been holding these patches. I'm also not 100% sure if we will need the longterm flag in the future.
This is also why I did not change the get_user_pages_longterm because we could be ripping this all out by the end of the year... (I hope. :-)
So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
[1] https://lkml.org/lkml/2019/2/11/1789
Quoted 5 lines
thanks,
Signature · 2 lines
John Hubbard NVIDIA
John Hubbard
reviewerRe: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Quoted 8 lines
Quoted 2 lines
[...]
It seems to me that the longterm vs. short-term is of questionable value.This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(
Yes, I agree. Below...
Quoted 11 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the FOLL_LONGTERM callers if and when we want to change anything. I just think also it's appopriate to go a bit further, and use FOLL_LONGTERM all by itself.
That's because in either design outcome, it's better that way:
-- If we keep the concept of "I'm a long-term gup call site", then FOLL_LONGTERM is just right. The gup API already has _fast and non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.
-- If we drop the concept, then you've already done part of the work, by removing the _longterm API variants.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 35 lines · John Hubbard
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Quoted 8 lines
Quoted 2 lines
[...]
It seems to me that the longterm vs. short-term is of questionable value.This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(Yes, I agree. Below...
Quoted 11 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the
FOLL_LONGTERMcallers if and when we want to change anything. I just think also it's appopriate to go a bit further, and useFOLL_LONGTERMall by itself.That's because in either design outcome, it's better that way:
Quoted 8 lines
-- If we keep the concept of "I'm a long-term gup call site", then
FOLL_LONGTERMis just right. The gup API already has_fastand non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.-- If we drop the concept, then you've already done part of the work, by removing the
_longtermAPI variants.
A problem I now see with the _longterm name is that it hides its true intent. It's really a "dax can't use page cache tricks to make it seem like this page is ok to access indefinitely, if the system needs this page back your pin would prevent the forward progress of the system state.". If the discussion results in a need to have an explicit file state (immutable or lease) then we'll continue to need a gup pin type distinction. If the discussion resolves to one of the silent options (fail truncate, lie about truncate) then FOLL_LONGTERM might be able to die at that point.
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
In reply to John HubbardQuoted 34 lines · John Hubbard
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Quoted 8 lines
Quoted 2 lines
[...]
It seems to me that the longterm vs. short-term is of questionable value.This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(Yes, I agree. Below...
Quoted 11 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the
FOLL_LONGTERMcallers if and when we want to change anything. I just think also it's appopriate to go a bit further, and useFOLL_LONGTERMall by itself.That's because in either design outcome, it's better that way:
Quoted 7 lines
-- If we keep the concept of "I'm a long-term gup call site", then
FOLL_LONGTERMis just right. The gup API already has_fastand non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.-- If we drop the concept, then you've already done part of the work, by removing the
_longtermAPI variants.
Fair enough. But to do that correctly I think we will need to convertget_user_pages_fast() to use flags as well. I have a version of this series which includes a patch does this, but the patch touched a lot of subsystems and a couple of different architectures...[1]
I can't test them all. If we want to go that way I'm up for submitting the patch... But if we remove longterm in the future we may be left with a get_user_pages_fast() which really only needs 1 flag. But perhaps overall we would be better off?
Ira
[1] mm/gup.c: Change GUP fast to use flags rather than write bool
To facilitate additional options to get_user_pages_fast change the singular write parameter to be the more generic gup_flags.
This patch currently does not change any functionality. New functionality will follow in subsequent patches.
Many of the get_user_pages_fast call sites were unchanged because they already used FOLL_WRITE or 0 as appropriate.
--- arch/mips/mm/gup.c | 11 ++++++----- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 ++-- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 ++-- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 ++++++------ arch/sh/mm/gup.c | 11 ++++++----- arch/sparc/mm/gup.c | 9 +++++---- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 ++- drivers/infiniband/hw/hfi1/user_pages.c | 3 ++- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 ++++-- drivers/platform/goldfish/goldfish_pipe.c | 3 ++- drivers/rapidio/devices/rio_mport_cdev.c | 4 +++- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 ++- drivers/staging/gasket/gasket_page_table.c | 4 ++-- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 ++- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 4 ++-- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +++++-- mm/gup.c | 10 +++++----- mm/util.c | 8 ++++---- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 ++- 36 files changed, 81 insertions(+), 65 deletions(-)
John Hubbard
reviewerRe: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 5 lines · Ira Weiny
Quoted 4 lines · John Hubbard
Quoted 3 lines · Ira Weiny
Quoted 2 lines · John Hubbard
Quoted 1 line · Jason Gunthorpe
On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
From: Ira Weiny <[email protected]>
[...]
Fair enough. But to do that correctly I think we will need to convert
get_user_pages_fast()to use flags as well. I have a version of this series which includes a patch does this, but the patch touched a lot of subsystems and a couple of different architectures...[1]I can't test them all. If we want to go that way I'm up for submitting the
I have a similar problem, and a similar list of call sites, for the put_user_pages() conversion, so that file list looks familiar. And the arch-specific gup implementations are about to complicate my life too. :)
Quoted 5 lines
patch... But if we remove longterm in the future we may be left with a
get_user_pages_fast()which really only needs 1 flag. But perhaps overall we would be better off?Ira
I certainly think so, yes.
thanks,
Signature · 2 lines
John Hubbard NVIDIA
Quoted 54 lines
[1]
mm/gup.c: Change GUP fast to use flags rather than write boolTo facilitate additional options to
get_user_pages_fastchange the singular write parameter to be the more genericgup_flags.This patch currently does not change any functionality. New functionality will follow in subsequent patches.
Many of the
get_user_pages_fastcall sites were unchanged because they already usedFOLL_WRITEor 0 as appropriate.Signed-off-by Ira Weiny <[email protected]>--- arch/mips/mm/gup.c | 11 ++++++----- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 ++-- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 ++-- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 ++++++------ arch/sh/mm/gup.c | 11 ++++++----- arch/sparc/mm/gup.c | 9 +++++---- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 ++- drivers/infiniband/hw/hfi1/user_pages.c | 3 ++- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 ++++-- drivers/platform/goldfish/goldfish_pipe.c | 3 ++- drivers/rapidio/devices/rio_mport_cdev.c | 4 +++- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 ++- drivers/staging/gasket/gasket_page_table.c | 4 ++-- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 ++- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 4 ++-- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +++++-- mm/gup.c | 10 +++++----- mm/util.c | 8 ++++---- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 ++- 36 files changed, 81 insertions(+), 65 deletions(-)
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
In reply to Ira WeinyQuoted 39 lines · Ira Weiny
Quoted 38 lines · John Hubbard
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Quoted 8 lines
Quoted 2 lines
[...]
It seems to me that the longterm vs. short-term is of questionable value.This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(Yes, I agree. Below...
Quoted 11 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the
FOLL_LONGTERMcallers if and when we want to change anything. I just think also it's appopriate to go a bit further, and useFOLL_LONGTERMall by itself.That's because in either design outcome, it's better that way:
is just right. The gup API already has
_fastand non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.
Quoted 6 lines
Quoted 1 line
the
_longtermAPI variants.Fair enough. But to do that correctly I think we will need to convert
get_user_pages_fast()to use flags as well. I have a version of this series which includes a patch does this, but the patch touched a lot of subsystems and a couple of different architectures...[1]
I think this should be done anyhow, it is trouble the two basically identical interfaces have different signatures. This already caused a bug in vfio..
I also wonder if someone should think about making fast into a flag too..
But I'm not sure when fast should be used vs when it shouldn't :(
Jason
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 48 lines · Jason Gunthorpe
Quoted 39 lines · Ira Weiny
Quoted 38 lines · John Hubbard
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line · On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
Quoted 8 lines
Quoted 2 lines
[...]
It seems to me that the longterm vs. short-term is of questionable value.This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(Yes, I agree. Below...
Quoted 11 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the
FOLL_LONGTERMcallers if and when we want to change anything. I just think also it's appopriate to go a bit further, and useFOLL_LONGTERMall by itself.That's because in either design outcome, it's better that way:
is just right. The gup API already has
_fastand non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.Quoted 6 lines
Quoted 1 line
the
_longtermAPI variants.Fair enough. But to do that correctly I think we will need to convert
get_user_pages_fast()to use flags as well. I have a version of this series which includes a patch does this, but the patch touched a lot of subsystems and a couple of different architectures...[1]
Quoted 8 lines
I think this should be done anyhow, it is trouble the two basically identical interfaces have different signatures. This already caused a bug in vfio..
I also wonder if someone should think about making fast into a flag too..
But I'm not sure when fast should be used vs when it shouldn't :(
Effectively fast should always be used just in case the user cares about performance. It's just that it may fail and need to fall back to requiring the vma.
Personally I thought RDMA memory registration is a one-time / upfront slow path so that non-fast-GUP is tolerable.
The workloads that *need* it are O_DIRECT users that can't tolerate a vma lookup on every I/O.
RE: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 69 lines · Jason Gunthorpe
Quoted 7 lines · Ira Weiny
Quoted 5 lines · John Hubbard
Quoted 4 lines · Ira Weiny
Quoted 3 lines · John Hubbard
Quoted 2 lines · Jason Gunthorpe
Quoted 1 line
On Mon, Feb 11, 2019 at 12:16:42PM -0800, [email protected]
Quoted 3 lines · wrote:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
From: Ira Weiny <[email protected]>
[...]
It seems to me that the longterm vs. short-term is of questionablevalue.
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 8 lines
This is exactly why I did not post this before. I've been waiting our other discussions on how GUP pins are going to be handled to play out. But with the netdev thread today[1] it seems like we need to make sure we have a "safe" fast variant for a while. Introducing
FOLL_LONGTERMseemed like the cleanest way to do that even if we will not need the distinction in the future... :-(Yes, I agree. Below...
Quoted 8 lines
[...]
This is also why I did not change theget_user_pages_longtermbecause we could be ripping this all out by the end of the year... (I hope. :-)So while this does "pollute" the GUP family of calls I'm hoping it is not forever.
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Ira
Yes, and to be clear, I think your patchset here is fine. It is easy to find the
FOLL_LONGTERMcallers if and when we want to change anything. I just think also it's appopriate to go a bit further, anduse
FOLL_LONGTERMall by itself.Quoted 14 lines
Quoted 7 lines
Quoted 7 lines
That's because in either design outcome, it's better that way:
is just right. The gup API already has
_fastand non-fast variants, and once you get past a couple, you end up with a multiplication of names that really work better as flags. We're there.Quoted 6 lines
Quoted 1 line
the
_longtermAPI variants.Fair enough. But to do that correctly I think we will need to convert
get_user_pages_fast()to use flags as well. I have a version of this series which includes a patch does this, but the patch touched a lot of subsystems and a couple of different architectures...[1]Quoted 8 lines
I think this should be done anyhow, it is trouble the two basically identical interfaces have different signatures. This already caused a bug in vfio..
I also wonder if someone should think about making fast into a flag too..
But I'm not sure when fast should be used vs when it shouldn't :(
Effectively fast should always be used just in case the user cares about performance. It's just that it may fail and need to fall back to requiring the vma.
Quoted 5 lines
Personally I thought RDMA memory registration is a one-time / upfront slow path so that non-fast-GUP is tolerable.
The workloads that *need* it are
O_DIRECTusers that can't tolerate a vma lookup on every I/O.
There are some users who need to [un]register memory more often. While not in the strict fast path these users would like the registrations to occur as fast as possible. I don't personally have the results but our OPA team did do performance tests on the GUP vs GUP fast and for the hfi1 case fast was better. I don't have any reason to believe that regular RDMA users would not also benefit.
Ira
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
In reply to Dan WilliamsQuoted 8 lines · Dan Williams
Quoted 4 lines
I also wonder if someone should think about making fast into a flag too..
But I'm not sure when fast should be used vs when it shouldn't :(
Effectively fast should always be used just in case the user cares about performance. It's just that it may fail and need to fall back to requiring the vma.
But the fall back / slow path is hidden inside the API, so when should the caller care?
ie when should the caller care to use gup_fast vs gup_unlocked? (the comments say they are the same, but this seems to be a mistake)
Based on some of the comments in the code it looks like this API is trying to convert itself into:
long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, bool *locked)
long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages)
(and maybe a FOLL_FAST if there is some reason we have _fast and _unlocked)
The reason I ask, is that if there is no reason for fast vs unlocked then maybe Ira should convert HFI to use gup_unlocked and move the 'fast' code into unlocked?
ie move incrementally closer to the desired end-state here.
Jason
Re: [PATCH 2/3] mm/gup: Introduce get_user_pages_fast_longterm()
Quoted 29 lines · Jason Gunthorpe
Quoted 8 lines · Dan Williams
Quoted 4 lines
I also wonder if someone should think about making fast into a flag too..
But I'm not sure when fast should be used vs when it shouldn't :(
Effectively fast should always be used just in case the user cares about performance. It's just that it may fail and need to fall back to requiring the vma.
But the fall back / slow path is hidden inside the API, so when should the caller care?
ie when should the caller care to use
gup_fastvsgup_unlocked? (the comments say they are the same, but this seems to be a mistake)Based on some of the comments in the code it looks like this API is trying to convert itself into:
long get_user_pages_locked(struct
task_struct*tsk, structmm_struct*mm,
unsigned long start, unsigned longnr_pages, unsigned intgup_flags, struct page **pages, structvm_area_struct**vmas, bool *locked)long get_user_pages_unlocked(struct
task_struct*tsk, structmm_struct*mm,
unsigned long start, unsigned longnr_pages, unsigned intgup_flags, struct page **pages)
Quoted 8 lines
(and maybe a
FOLL_FASTif there is some reason we have_fastand_unlocked)The reason I ask, is that if there is no reason for fast vs unlocked then maybe Ira should convert HFI to use
gup_unlockedand move the 'fast' code into unlocked?ie move incrementally closer to the desired end-state here.
If the pages are not in the page tables then fast is probably going to be slightly slower because it will have to fall back after walking the tables and finding something missing.
For PSM2 (MPI) applications are performance improvement was probably because the memory in question was in the page tables and very much in use.
Ira
Quoted 2 lines
Jason
PATCH 3/3 IB/HFI1: Use new get_user_pages_fast_longterm()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new get_user_pages_fast_longterm() call to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/hfi1/user_pages.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/hw/hfi1/user_pages.c +1 -1
@@ -105,7 +105,7 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np {int ret; - ret =get_user_pages_fast (vaddr, npages, writable, pages); + ret =get_user_pages_fast_longterm (vaddr, npages, writable, pages);if (ret <0 )return ret;
Signature · 1 line
2.20.1
Quoted 4 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()
Out of curiosity, are you planning on having all rdma drivers use get_user_pages_fast_longterm()? Ie:
hw/mthca/mthca_memfree.c: ret = get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages); hw/qib/qib_user_sdma.c: ret = get_user_pages_fast(addr, j, 0, pages);
Thanks,
Davidlohr
Quoted 10 lines · Davidlohr Bueso
Quoted 4 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()Out of curiosity, are you planning on having all rdma drivers use
get_user_pages_fast_longterm()? Ie:
hw/mthca/mthca_memfree.c: ret =get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages);
This one is certainly a mistake - this should be done with a umem.
Jason
Quoted 7 lines · Jason Gunthorpe
Quoted 6 lines · Davidlohr Bueso
Quoted 4 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()
Quoted 6 lines
Quoted 4 lines
Out of curiosity, are you planning on having all rdma drivers use
get_user_pages_fast_longterm()? Ie:
hw/mthca/mthca_memfree.c: ret =get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages);This one is certainly a mistake - this should be done with a umem.
It looks like this is mapping a page allocated by user space for a doorbell?!?! And that this is supporting the old memory free cards. I remember that these cards used system memory instead of memory on the cards but why it expects user space to allocate that memory and how it all works is way too old for me to even try to remember.
This does not seem to be allocating memory regions. Jason, do you want a patch to just convert these calls and consider it legacy code?
Ira
Quoted 2 lines
Jason
Quoted 11 lines · Ira Weiny
Quoted 10 lines · Jason Gunthorpe
Quoted 9 lines · Davidlohr Bueso
Quoted 4 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()Out of curiosity, are you planning on having all rdma drivers use
get_user_pages_fast_longterm()? Ie:
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
hw/mthca/mthca_memfree.c: ret =get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages);This one is certainly a mistake - this should be done with a umem.
It looks like this is mapping a page allocated by user space for a doorbell?!?!
Many drivers do this, the 'doorbell' is a PCI -> CPU thing of some sort
Quoted 3 lines
This does not seem to be allocating memory regions. Jason, do you want a patch to just convert these calls and consider it legacy code?
It needs to use umem like all the other drivers on this path. Otherwise it doesn't get the page pinning logic right
There is also something else rotten with these longterm callsites, they seem to have very different ideas how to handle RLIMIT_MEMLOCK.
ie vfio doesn't even touch pinned_vm.. and rdma is applying RLIMIT_MEMLOCK to mm->pinned_vm, while vfio is using locked_vm.. No idea which is right, but they should be the same, and this pattern should probably be in core code someplace.
Jason
Quoted 13 lines · Ira Weiny
Quoted 8 lines · Jason Gunthorpe
Quoted 7 lines · Davidlohr Bueso
Quoted 6 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Quoted 4 lines
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Out of curiosity, are you planning on having all rdma drivers use
get_user_pages_fast_longterm()? Ie:
hw/mthca/mthca_memfree.c: ret = get_user_pages_fast(uaddr &
Quoted 8 lines
PAGE_MASK, 1,FOLL_WRITE, pages);Quoted 5 lines
Quoted 2 lines
This one is certainly a mistake - this should be done with a umem.
It looks like this is mapping a page allocated by user space for a doorbell?!?!
Many drivers do this, the 'doorbell' is a PCI -> CPU thing of some sort
My surprise is why does _userspace_ allocate this memory?
Quoted 6 lines
Quoted 3 lines
This does not seem to be allocating memory regions. Jason, do you want a patch to just convert these calls and consider it legacy code?
It needs to use umem like all the other drivers on this path. Otherwise it doesn't get the page pinning logic right
Not sure what you mean regarding the pinning logic?
Quoted 8 lines
There is also something else rotten with these longterm callsites, they seem to have very different ideas how to handle
RLIMIT_MEMLOCK.ie vfio doesn't even touch
pinned_vm.. and rdma is applyingRLIMIT_MEMLOCKtomm->pinned_vm, while vfio is usinglocked_vm.. No idea which is right, but they should be the same, and this pattern should probably be in core code someplace.
Neither do I. But AFAIK pinned_vm is a subset of locked_vm.
So should we be accounting both of the counters?
Ira
Quoted 3 lines · Ira
Quoted 1 line
Many drivers do this, the 'doorbell' is a PCI -> CPU thing of some sort
My surprise is why does _userspace_ allocate this memory?
Well, userspace needs to read the memory, so either userpace allocates it and the kernel GUP's it, or userspace mmap's a kernel page which was DMA mapped.
The GUP version lets the doorbells have lower alignment than a PAGE, and thes RDMA drivers hard requires GUP->DMA to function..
So why not use a umem here? It already has to work.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
This does not seem to be allocating memory regions. Jason, do you want a patch to just convert these calls and consider it legacy code?
It needs to use umem like all the other drivers on this path. Otherwise it doesn't get the page pinning logic right
Not sure what you mean regarding the pinning logic?
The RLIMIT_MEMLOCK stuff and so on.
Quoted 10 lines
Quoted 8 lines
There is also something else rotten with these longterm callsites, they seem to have very different ideas how to handle
RLIMIT_MEMLOCK.ie vfio doesn't even touch
pinned_vm.. and rdma is applyingRLIMIT_MEMLOCKtomm->pinned_vm, while vfio is usinglocked_vm.. No idea which is right, but they should be the same, and this pattern should probably be in core code someplace.Neither do I. But AFAIK
pinned_vmis a subset oflocked_vm.
I thought so..
Quoted 1 line
So should we be accounting both of the counters?
Someone should check :)
Since we don't increment locked_vm when we increment pinned_vm and vfio only checke RLIMIT_MEMLOCK against locked_vm one can certainly exceed the limit by mixing and matching RDMA and VFIO pins in the same process. Sure seems like there is a bug somewhere here.
Jason
Quoted 11 lines · Davidlohr Bueso
Quoted 4 lines · On Mon, 11 Feb 2019, [email protected] wrote:
Ira Weiny (3):
mm/gup: Change "write" parameter to flags
mm/gup: Introduceget_user_pages_fast_longterm()
IB/HFI1: Use newget_user_pages_fast_longterm()Out of curiosity, are you planning on having all rdma drivers use
get_user_pages_fast_longterm()? Ie:hw/mthca/mthca_memfree.c: ret = get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages); hw/qib/qib_user_sdma.c: ret = get_user_pages_fast(addr, j, 0, pages);
I missed that when I change the other qib call to longterm... :-(
Yes both of these should be changed. Although I need to look into Jasons comment WRT the mthca call.
Ira
Quoted 3 lines
Thanks,
Davidlohr
Quoted 8 lines · On Mon, Feb 11, 2019 at 12:16:40PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
HFI1 uses
get_user_pages_fast()due to it performance advantages. Like RDMA, HFI1 pages can be held for a significant time. Butget_user_pages_fast()does not protect against mapping of FS DAX pages.
If HFI1 can use the _fast varient, can't all the general RDMA stuff use it too?
What is the guidance on when fast vs not fast should be use?
Jason
Quoted 7 lines · On Mon, Feb 11, 2019 at 12:16:40PM -0800, [email protected] wrote:
Quoted 5 lines
From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
Quoted 8 lines
Quoted 4 lines
HFI1 uses
get_user_pages_fast()due to it performance advantages. Like RDMA,
HFI1 pages can be held for a significant time. Butget_user_pages_fast()does not protect against mapping of FS DAX pages.If HFI1 can use the
_fastvarient, can't all the general RDMA stuff use it too?What is the guidance on when fast vs not fast should be use?
Right now it can't because it holds mmap_sem across the call. Once Shiraz's patches are accepted removing the umem->hugetlb flag I think we can change umem.c.
Also, it specifies FOLL_FORCE which can't currently be specified with gup fast. One idea I had was to change get_user_pages_fast() to use gup_flags instead of a single write flag. But that proved to be a very big cosmetic change across a lot of callers so I went this way.
Ira
Quoted 2 lines
Jason
Quoted 12 lines · Ira
Quoted 12 lines · On Mon, Feb 11, 2019 at 12:16:40PM -0800, [email protected] wrote:
Quoted 9 lines
From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
HFI1 uses
get_user_pages_fast()due to it performance advantages. Like RDMA,
HFI1 pages can be held for a significant time. Butget_user_pages_fast()does not protect against mapping of FS DAX pages.
Quoted 7 lines
Quoted 3 lines
If HFI1 can use the
_fastvarient, can't all the general RDMA stuff use it too?What is the guidance on when fast vs not fast should be use?
Right now it can't because it holds
mmap_semacross the call. Once Shiraz's patches are accepted removing theumem->hugetlbflag I think we can changeumem.c.
Okay, that make sense, we should change it when Shiraz's patches are merged
Quoted 5 lines
Also, it specifies
FOLL_FORCEwhich can't currently be specified with gup fast. One idea I had was to changeget_user_pages_fast()to usegup_flagsinstead of a single write flag. But that proved to be a very big cosmetic change across a lot of callers so I went this way.
I think you should do it.. :)
Jason
v2 cover Add FOLL_LONGTERM to GUP fast and use it
In reply to [email protected]From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
HFI1, qib, and mthca, use get_user_pages_fast() due to it performance advantages. These pages can be held for a significant time. But get_user_pages_fast() does not protect against mapping of FS DAX pages.
Introduce FOLL_LONGTERM and use this flag in get_user_pages_fast() which retains the performance while also adding the FS DAX checks. XDP has also shown interest in using this functionality.[2]
In addition we change get_user_pages() to use the new FOLL_LONGTERM flag and remove the specialized get_user_pages_longterm call.
[1] https://lkml.org/lkml/2019/2/11/237
[2] https://lkml.org/lkml/2019/2/11/1789
Ira Weiny (7):
mm/gup: Replace get_user_pages_longterm() with FOLL_LONGTERM mm/gup: Change write parameter to flags in fast walk mm/gup: Change GUP fast to use flags rather than a write 'bool' mm/gup: Add FOLL_LONGTERM capability to GUP fast IB/hfi1: Use the new FOLL_LONGTERM flag to get_user_pages_fast() IB/qib: Use the new FOLL_LONGTERM flag to get_user_pages_fast() IB/mthca: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
arch/mips/mm/gup.c | 11 +- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 +- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 +- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 +- arch/sh/mm/gup.c | 11 +- arch/sparc/mm/gup.c | 9 +- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 +- drivers/infiniband/core/umem.c | 5 +- drivers/infiniband/hw/hfi1/user_pages.c | 5 +- drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +- drivers/infiniband/hw/qib/qib_user_pages.c | 8 +- drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +- drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 6 +- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 +- drivers/platform/goldfish/goldfish_pipe.c | 3 +- drivers/rapidio/devices/rio_mport_cdev.c | 4 +- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 +- drivers/staging/gasket/gasket_page_table.c | 4 +- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 +- drivers/vfio/vfio_iommu_type1.c | 3 +- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 17 +- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +- mm/gup.c | 220 ++++++++++++-------- mm/gup_benchmark.c | 5 +- mm/util.c | 8 +- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 +- 44 files changed, 232 insertions(+), 180 deletions(-)
Signature · 1 line
2.20.1
PATCH v2 1/7 mm/gup: Replace get_user_pages_longterm() with FOLL_LONGTERM
From: Ira Weiny <[email protected]>
Rather than have a separate get_user_pages_longterm() call, introduce FOLL_LONGTERM and change the longterm callers to use it.
This patch does not change any functionality.
FOLL_LONGTERM can only be supported with get_user_pages() as it requires vmas to determine if DAX is in use.
--- drivers/infiniband/core/umem.c | 5 +- drivers/infiniband/hw/qib/qib_user_pages.c | 8 +- drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 6 +- drivers/vfio/vfio_iommu_type1.c | 3 +- include/linux/mm.h | 13 +- mm/gup.c | 138 ++++++++++++--------- mm/gup_benchmark.c | 5 +- 8 files changed, 101 insertions(+), 86 deletions(-)
drivers/infiniband/core/umem.c +3 -2
@@ -185,10 +185,11 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,while (npages) {down_read (&mm->mmap_sem); - ret =get_user_pages_longterm (cur_base, + ret =get_user_pages (cur_base, min_t(unsigned long, npages,PAGE_SIZE /sizeof (struct page *)), - gup_flags, page_list, vma_list); + gup_flags |FOLL_LONGTERM , + page_list, vma_list);if (ret <0 ) {up_read (&mm->mmap_sem); goto umem_release;
drivers/infiniband/hw/qib/qib_user_pages.c +4 -4
@@ -114,10 +114,10 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages,down_read (¤t->mm->mmap_sem);for (got =0 ; got < num_pages; got += ret) { - ret =get_user_pages_longterm (start_page + got *PAGE_SIZE , - num_pages - got, -FOLL_WRITE |FOLL_FORCE , - p + got,NULL ); + ret =get_user_pages (start_page + got *PAGE_SIZE , + num_pages - got, +FOLL_LONGTERM |FOLL_WRITE |FOLL_FORCE , + p + got,NULL );if (ret <0 ) {up_read (¤t->mm->mmap_sem); goto bail_release;
drivers/infiniband/hw/usnic/usnic_uiom.c +5 -4
@@ -143,10 +143,11 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, ret =0 ;while (npages) { - ret =get_user_pages_longterm (cur_base, - min_t(unsigned long, npages, -PAGE_SIZE /sizeof (struct page *)), - gup_flags, page_list,NULL ); + ret =get_user_pages (cur_base, + min_t(unsigned long, npages, +PAGE_SIZE /sizeof (struct page *)), + gup_flags |FOLL_LONGTERM , + page_list,NULL );if (ret <0 ) goto out;
drivers/media/v4l2-core/videobuf-dma-sg.c +3 -3
@@ -186,12 +186,12 @@ static int videobuf_dma_init_user_locked(struct videobuf_dmabuf *dma,dprintk (1 ,"init user [0x%lx+0x%lx => %d pages]\n" , data, size, dma->nr_pages); - err =get_user_pages_longterm (data &PAGE_MASK , dma->nr_pages, - flags, dma->pages,NULL ); + err =get_user_pages (data &PAGE_MASK , dma->nr_pages, + flags |FOLL_LONGTERM , dma->pages,NULL );if (err != dma->nr_pages) { dma->nr_pages = (err >=0 ) ? err :0 ; -dprintk (1 ,"get_user_pages_longterm: err=%d [%d]\n" , err, +dprintk (1 ,"get_user_pages: err=%d [%d]\n" , err, dma->nr_pages);return err <0 ? err : -EINVAL ; }
drivers/vfio/vfio_iommu_type1.c +2 -1
@@ -351,7 +351,8 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,down_read (&mm->mmap_sem);if (mm == current->mm) { - ret =get_user_pages_longterm (vaddr,1 , flags, page, vmas); + ret =get_user_pages (vaddr,1 , flags |FOLL_LONGTERM , page, + vmas); }else { ret =get_user_pages_remote (NULL , mm, vaddr,1 , flags, page, vmas,NULL );
include/linux/mm.h +1 -12
@@ -1536,18 +1536,6 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);long get_user_pages_unlocked (unsigned long start,unsigned long nr_pages,struct page **pages,unsigned int gup_flags); -#ifdef CONFIG_FS_DAX -long get_user_pages_longterm (unsigned long start,unsigned long nr_pages, -unsigned int gup_flags,struct page **pages, -struct vm_area_struct **vmas); -#else -static inline long get_user_pages_longterm (unsigned long start, -unsigned long nr_pages,unsigned int gup_flags, -struct page **pages,struct vm_area_struct **vmas) -{ -return get_user_pages (start, nr_pages, gup_flags, pages, vmas); -} -#endif /* CONFIG_FS_DAX */ int get_user_pages_fast (unsigned long start,int nr_pages,int write,struct page **pages);
@@ -2615,6 +2603,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */ static inline int vm_fault_to_errno (vm_fault_t vm_fault,int foll_flags) {
mm/gup.c +80 -58
167 lines not on this page · raw message
mm/gup_benchmark.c +3 -2
@@ -54,8 +54,9 @@ static int __gup_benchmark_ioctl(unsigned int cmd, pages + i);break ;case GUP_LONGTERM_BENCHMARK : - nr =get_user_pages_longterm (addr, nr, gup->flags &1 , - pages + i,NULL ); + nr =get_user_pages (addr, nr, + (gup->flags &1 ) |FOLL_LONGTERM , + pages + i,NULL );break ;case GUP_BENCHMARK : nr = get_user_pages(addr, nr, gup->flags & 1, pages + i,
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 1/7 mm/gup: Replace get_user_pages_longterm() with FOLL_LONGTERM
In reply to [email protected]From: Ira Weiny <[email protected]>
Rather than have a separate get_user_pages_longterm() call, introduce FOLL_LONGTERM and change the longterm callers to use it.
This patch does not change any functionality.
FOLL_LONGTERM can only be supported with get_user_pages() as it requires vmas to determine if DAX is in use.
--- drivers/infiniband/core/umem.c | 5 +- drivers/infiniband/hw/qib/qib_user_pages.c | 8 +- drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 6 +- drivers/vfio/vfio_iommu_type1.c | 3 +- include/linux/mm.h | 13 +- mm/gup.c | 138 ++++++++++++--------- mm/gup_benchmark.c | 5 +- 8 files changed, 101 insertions(+), 86 deletions(-)
drivers/infiniband/core/umem.c +3 -2
@@ -185,10 +185,11 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,while (npages) {down_read (&mm->mmap_sem); - ret =get_user_pages_longterm (cur_base, + ret =get_user_pages (cur_base, min_t(unsigned long, npages,PAGE_SIZE /sizeof (struct page *)), - gup_flags, page_list, vma_list); + gup_flags |FOLL_LONGTERM , + page_list, vma_list);if (ret <0 ) {up_read (&mm->mmap_sem); goto umem_release;
drivers/infiniband/hw/qib/qib_user_pages.c +4 -4
@@ -114,10 +114,10 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages,down_read (¤t->mm->mmap_sem);for (got =0 ; got < num_pages; got += ret) { - ret =get_user_pages_longterm (start_page + got *PAGE_SIZE , - num_pages - got, -FOLL_WRITE |FOLL_FORCE , - p + got,NULL ); + ret =get_user_pages (start_page + got *PAGE_SIZE , + num_pages - got, +FOLL_LONGTERM |FOLL_WRITE |FOLL_FORCE , + p + got,NULL );if (ret <0 ) {up_read (¤t->mm->mmap_sem); goto bail_release;
drivers/infiniband/hw/usnic/usnic_uiom.c +5 -4
@@ -143,10 +143,11 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, ret =0 ;while (npages) { - ret =get_user_pages_longterm (cur_base, - min_t(unsigned long, npages, -PAGE_SIZE /sizeof (struct page *)), - gup_flags, page_list,NULL ); + ret =get_user_pages (cur_base, + min_t(unsigned long, npages, +PAGE_SIZE /sizeof (struct page *)), + gup_flags |FOLL_LONGTERM , + page_list,NULL );if (ret <0 ) goto out;
drivers/media/v4l2-core/videobuf-dma-sg.c +3 -3
@@ -186,12 +186,12 @@ static int videobuf_dma_init_user_locked(struct videobuf_dmabuf *dma,dprintk (1 ,"init user [0x%lx+0x%lx => %d pages]\n" , data, size, dma->nr_pages); - err =get_user_pages_longterm (data &PAGE_MASK , dma->nr_pages, - flags, dma->pages,NULL ); + err =get_user_pages (data &PAGE_MASK , dma->nr_pages, + flags |FOLL_LONGTERM , dma->pages,NULL );if (err != dma->nr_pages) { dma->nr_pages = (err >=0 ) ? err :0 ; -dprintk (1 ,"get_user_pages_longterm: err=%d [%d]\n" , err, +dprintk (1 ,"get_user_pages: err=%d [%d]\n" , err, dma->nr_pages);return err <0 ? err : -EINVAL ; }
drivers/vfio/vfio_iommu_type1.c +2 -1
@@ -351,7 +351,8 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,down_read (&mm->mmap_sem);if (mm == current->mm) { - ret =get_user_pages_longterm (vaddr,1 , flags, page, vmas); + ret =get_user_pages (vaddr,1 , flags |FOLL_LONGTERM , page, + vmas); }else { ret =get_user_pages_remote (NULL , mm, vaddr,1 , flags, page, vmas,NULL );
include/linux/mm.h +1 -12
@@ -1536,18 +1536,6 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);long get_user_pages_unlocked (unsigned long start,unsigned long nr_pages,struct page **pages,unsigned int gup_flags); -#ifdef CONFIG_FS_DAX -long get_user_pages_longterm (unsigned long start,unsigned long nr_pages, -unsigned int gup_flags,struct page **pages, -struct vm_area_struct **vmas); -#else -static inline long get_user_pages_longterm (unsigned long start, -unsigned long nr_pages,unsigned int gup_flags, -struct page **pages,struct vm_area_struct **vmas) -{ -return get_user_pages (start, nr_pages, gup_flags, pages, vmas); -} -#endif /* CONFIG_FS_DAX */ int get_user_pages_fast (unsigned long start,int nr_pages,int write,struct page **pages);
@@ -2615,6 +2603,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ #define FOLL_COW 0x4000/* internal GUP flag */ #define FOLL_ANON 0x8000/* don't do file mappings */ +#define FOLL_LONGTERM 0x10000/* mapping is intended for a long term pin */ static inline int vm_fault_to_errno (vm_fault_t vm_fault,int foll_flags) {
mm/gup.c +80 -58
167 lines not on this page · raw message
mm/gup_benchmark.c +3 -2
@@ -54,8 +54,9 @@ static int __gup_benchmark_ioctl(unsigned int cmd, pages + i);break ;case GUP_LONGTERM_BENCHMARK : - nr =get_user_pages_longterm (addr, nr, gup->flags &1 , - pages + i,NULL ); + nr =get_user_pages (addr, nr, + (gup->flags &1 ) |FOLL_LONGTERM , + pages + i,NULL );break ;case GUP_BENCHMARK : nr = get_user_pages(addr, nr, gup->flags & 1, pages + i,
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 2/7 mm/gup: Change write parameter to flags in fast walk
In reply to [email protected]From: Ira Weiny <[email protected]>
In order to support more options in the GUP fast walk, change the write parameter to flags throughout the call stack.
This patch does not change functionality and passes FOLL_WRITE where write was previously used.
--- mm/gup.c | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-)
mm/gup.c +26 -26
@@ -1417,7 +1417,7 @@ static void undo_dev_pagemap(int *nr, int nr_start, struct page **pages)#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {struct dev_pagemap *pgmap =NULL ;int nr_start = *nr, ret =0 ;
@@ -1435,7 +1435,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,if (pte_protnone (pte)) goto pte_unmap; -if (!pte_access_permitted (pte, write)) +if (!pte_access_permitted (pte, flags &FOLL_WRITE )) goto pte_unmap;if (pte_devmap (pte)) {
@@ -1487,7 +1487,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,* useful to have gup_huge_pmd even if we can't operate on ptes. */ static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {return 0 ; }
@@ -1570,12 +1570,12 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,#endif static int gup_huge_pmd (pmd_t orig,pmd_t *pmdp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pmd_access_permitted (orig, write)) +if (!pmd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pmd_devmap (orig))
@@ -1608,12 +1608,12 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, }static int gup_huge_pud (pud_t orig,pud_t *pudp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pud_access_permitted (orig, write)) +if (!pud_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pud_devmap (orig))
@@ -1646,13 +1646,13 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, }static int gup_huge_pgd (pgd_t orig,pgd_t *pgdp,unsigned long addr, -unsigned long end,int write, +unsigned long end,unsigned int flags,struct page **pages,int *nr) {int refs;struct page *head, *page; -if (!pgd_access_permitted (orig, write)) +if (!pgd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;BUILD_BUG_ON (pgd_devmap (orig));
@@ -1683,7 +1683,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, }static int gup_pmd_range (pud_t pud,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pmd_t *pmdp;
@@ -1705,7 +1705,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,if (pmd_protnone (pmd))return 0 ; -if (!gup_huge_pmd (pmd, pmdp, addr, next, write, +if (!gup_huge_pmd (pmd, pmdp, addr, next, flags, pages, nr))return 0 ;
@@ -1715,9 +1715,9 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,* pmd format and THP pmd format */ if (!gup_huge_pd (__hugepd (pmd_val (pmd)), addr, -PMD_SHIFT , next, write, pages, nr)) +PMD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pte_range (pmd, addr, next, write, pages, nr)) + }else if (!gup_pte_range (pmd, addr, next, flags, pages, nr))return 0 ; }while (pmdp++, addr = next, addr != end);
@@ -1725,7 +1725,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, }static int gup_pud_range (p4d_t p4d,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pud_t *pudp;
@@ -1738,14 +1738,14 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,if (pud_none (pud))return 0 ;if (unlikely (pud_huge (pud))) { -if (!gup_huge_pud (pud, pudp, addr, next, write, +if (!gup_huge_pud (pud, pudp, addr, next, flags, pages, nr))return 0 ; }else if (unlikely (is_hugepd (__hugepd (pud_val (pud))))) {if (!gup_huge_pd (__hugepd (pud_val (pud)), addr, -PUD_SHIFT , next, write, pages, nr)) +PUD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pmd_range (pud, addr, next, write, pages, nr)) + }else if (!gup_pmd_range (pud, addr, next, flags, pages, nr))return 0 ; }while (pudp++, addr = next, addr != end);
@@ -1753,7 +1753,7 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end, }static int gup_p4d_range (pgd_t pgd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;p4d_t *p4dp;
@@ -1768,9 +1768,9 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,BUILD_BUG_ON (p4d_huge (p4d));if (unlikely (is_hugepd (__hugepd (p4d_val (p4d))))) {if (!gup_huge_pd (__hugepd (p4d_val (p4d)), addr, -P4D_SHIFT , next, write, pages, nr)) +P4D_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pud_range (p4d, addr, next, write, pages, nr)) + }else if (!gup_pud_range (p4d, addr, next, flags, pages, nr))return 0 ; }while (p4dp++, addr = next, addr != end);
@@ -1778,7 +1778,7 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end, }static void gup_pgd_range (unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pgd_t *pgdp;
@@ -1791,14 +1791,14 @@ static void gup_pgd_range(unsigned long addr, unsigned long end,if (pgd_none (pgd))return ;if (unlikely (pgd_huge (pgd))) { -if (!gup_huge_pgd (pgd, pgdp, addr, next, write, +if (!gup_huge_pgd (pgd, pgdp, addr, next, flags, pages, nr))return ; }else if (unlikely (is_hugepd (__hugepd (pgd_val (pgd))))) {if (!gup_huge_pd (__hugepd (pgd_val (pgd)), addr, -PGDIR_SHIFT , next, write, pages, nr)) +PGDIR_SHIFT , next, flags, pages, nr))return ; - }else if (!gup_p4d_range (pgd, addr, next, write, pages, nr)) + }else if (!gup_p4d_range (pgd, addr, next, flags, pages, nr))return ; }while (pgdp++, addr = next, addr != end); }
@@ -1852,7 +1852,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_save (flags); -gup_pgd_range (start, end, write, pages, &nr); +gup_pgd_range (start, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_restore (flags); }
@@ -1894,7 +1894,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_disable (); -gup_pgd_range (addr, end, write, pages, &nr); +gup_pgd_range (addr, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_enable (); ret = nr; }
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 2/7 mm/gup: Change write parameter to flags in fast walk
In reply to [email protected]From: Ira Weiny <[email protected]>
In order to support more options in the GUP fast walk, change the write parameter to flags throughout the call stack.
This patch does not change functionality and passes FOLL_WRITE where write was previously used.
--- mm/gup.c | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-)
mm/gup.c +26 -26
@@ -1417,7 +1417,7 @@ static void undo_dev_pagemap(int *nr, int nr_start, struct page **pages)#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {struct dev_pagemap *pgmap =NULL ;int nr_start = *nr, ret =0 ;
@@ -1435,7 +1435,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,if (pte_protnone (pte)) goto pte_unmap; -if (!pte_access_permitted (pte, write)) +if (!pte_access_permitted (pte, flags &FOLL_WRITE )) goto pte_unmap;if (pte_devmap (pte)) {
@@ -1487,7 +1487,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,* useful to have gup_huge_pmd even if we can't operate on ptes. */ static int gup_pte_range (pmd_t pmd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {return 0 ; }
@@ -1570,12 +1570,12 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,#endif static int gup_huge_pmd (pmd_t orig,pmd_t *pmdp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pmd_access_permitted (orig, write)) +if (!pmd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pmd_devmap (orig))
@@ -1608,12 +1608,12 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, }static int gup_huge_pud (pud_t orig,pud_t *pudp,unsigned long addr, -unsigned long end,int write,struct page **pages,int *nr) +unsigned long end,unsigned int flags,struct page **pages,int *nr) {struct page *head, *page;int refs; -if (!pud_access_permitted (orig, write)) +if (!pud_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;if (pud_devmap (orig))
@@ -1646,13 +1646,13 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, }static int gup_huge_pgd (pgd_t orig,pgd_t *pgdp,unsigned long addr, -unsigned long end,int write, +unsigned long end,unsigned int flags,struct page **pages,int *nr) {int refs;struct page *head, *page; -if (!pgd_access_permitted (orig, write)) +if (!pgd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ;BUILD_BUG_ON (pgd_devmap (orig));
@@ -1683,7 +1683,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, }static int gup_pmd_range (pud_t pud,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pmd_t *pmdp;
@@ -1705,7 +1705,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,if (pmd_protnone (pmd))return 0 ; -if (!gup_huge_pmd (pmd, pmdp, addr, next, write, +if (!gup_huge_pmd (pmd, pmdp, addr, next, flags, pages, nr))return 0 ;
@@ -1715,9 +1715,9 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,* pmd format and THP pmd format */ if (!gup_huge_pd (__hugepd (pmd_val (pmd)), addr, -PMD_SHIFT , next, write, pages, nr)) +PMD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pte_range (pmd, addr, next, write, pages, nr)) + }else if (!gup_pte_range (pmd, addr, next, flags, pages, nr))return 0 ; }while (pmdp++, addr = next, addr != end);
@@ -1725,7 +1725,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, }static int gup_pud_range (p4d_t p4d,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pud_t *pudp;
@@ -1738,14 +1738,14 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,if (pud_none (pud))return 0 ;if (unlikely (pud_huge (pud))) { -if (!gup_huge_pud (pud, pudp, addr, next, write, +if (!gup_huge_pud (pud, pudp, addr, next, flags, pages, nr))return 0 ; }else if (unlikely (is_hugepd (__hugepd (pud_val (pud))))) {if (!gup_huge_pd (__hugepd (pud_val (pud)), addr, -PUD_SHIFT , next, write, pages, nr)) +PUD_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pmd_range (pud, addr, next, write, pages, nr)) + }else if (!gup_pmd_range (pud, addr, next, flags, pages, nr))return 0 ; }while (pudp++, addr = next, addr != end);
@@ -1753,7 +1753,7 @@ static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end, }static int gup_p4d_range (pgd_t pgd,unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;p4d_t *p4dp;
@@ -1768,9 +1768,9 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,BUILD_BUG_ON (p4d_huge (p4d));if (unlikely (is_hugepd (__hugepd (p4d_val (p4d))))) {if (!gup_huge_pd (__hugepd (p4d_val (p4d)), addr, -P4D_SHIFT , next, write, pages, nr)) +P4D_SHIFT , next, flags, pages, nr))return 0 ; - }else if (!gup_pud_range (p4d, addr, next, write, pages, nr)) + }else if (!gup_pud_range (p4d, addr, next, flags, pages, nr))return 0 ; }while (p4dp++, addr = next, addr != end);
@@ -1778,7 +1778,7 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end, }static void gup_pgd_range (unsigned long addr,unsigned long end, -int write,struct page **pages,int *nr) +unsigned int flags,struct page **pages,int *nr) {unsigned long next;pgd_t *pgdp;
@@ -1791,14 +1791,14 @@ static void gup_pgd_range(unsigned long addr, unsigned long end,if (pgd_none (pgd))return ;if (unlikely (pgd_huge (pgd))) { -if (!gup_huge_pgd (pgd, pgdp, addr, next, write, +if (!gup_huge_pgd (pgd, pgdp, addr, next, flags, pages, nr))return ; }else if (unlikely (is_hugepd (__hugepd (pgd_val (pgd))))) {if (!gup_huge_pd (__hugepd (pgd_val (pgd)), addr, -PGDIR_SHIFT , next, write, pages, nr)) +PGDIR_SHIFT , next, flags, pages, nr))return ; - }else if (!gup_p4d_range (pgd, addr, next, write, pages, nr)) + }else if (!gup_p4d_range (pgd, addr, next, flags, pages, nr))return ; }while (pgdp++, addr = next, addr != end); }
@@ -1852,7 +1852,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_save (flags); -gup_pgd_range (start, end, write, pages, &nr); +gup_pgd_range (start, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_restore (flags); }
@@ -1894,7 +1894,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_disable (); -gup_pgd_range (addr, end, write, pages, &nr); +gup_pgd_range (addr, end, write ?FOLL_WRITE :0 , pages, &nr);local_irq_enable (); ret = nr; }
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 3/7 mm/gup: Change GUP fast to use flags rather than a write 'bool'
In reply to [email protected]From: Ira Weiny <[email protected]>
To facilitate additional options to get_user_pages_fast() change the singular write parameter to be gup_flags.
This patch does not change any functionality. New functionality will follow in subsequent patches.
Some of the get_user_pages_fast() call sites were unchanged because they already passed FOLL_WRITE or 0 for the write parameter.
--- arch/mips/mm/gup.c | 11 ++++++----- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 ++-- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 ++-- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 ++++++------ arch/sh/mm/gup.c | 11 ++++++----- arch/sparc/mm/gup.c | 9 +++++---- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 ++- drivers/infiniband/hw/hfi1/user_pages.c | 3 ++- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 ++++-- drivers/platform/goldfish/goldfish_pipe.c | 3 ++- drivers/rapidio/devices/rio_mport_cdev.c | 4 +++- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 ++- drivers/staging/gasket/gasket_page_table.c | 4 ++-- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 ++- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 4 ++-- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +++++-- mm/gup.c | 10 +++++----- mm/util.c | 8 ++++---- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 ++- 36 files changed, 81 insertions(+), 65 deletions(-)
arch/mips/mm/gup.c +6 -5
@@ -235,7 +235,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -247,8 +247,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -273,7 +273,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);local_irq_enable ();
@@ -289,7 +290,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, pages += nr; ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , - pages, write ?FOLL_WRITE :0 ); + pages, gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/powerpc/kvm/book3s_64_mmu_hv.c +2 -2
@@ -582,7 +582,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,/* If writing != 0, then the HPTE must allow writing, if we get here */ write_ok = writing; hva =gfn_to_hva_memslot (memslot, gfn); - npages =get_user_pages_fast (hva,1 , writing, pages); + npages =get_user_pages_fast (hva,1 , writing ?FOLL_WRITE :0 , pages);if (npages <1 ) {/* Check if it's an I/O mapping */ down_read (¤t->mm->mmap_sem);
@@ -1175,7 +1175,7 @@ void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,if (!memslot || (memslot->flags &KVM_MEMSLOT_INVALID )) goto err; hva =gfn_to_hva_memslot (memslot, gfn); - npages =get_user_pages_fast (hva,1 ,1 , pages); + npages =get_user_pages_fast (hva,1 ,FOLL_WRITE , pages);if (npages <1 ) goto err; page = pages[0 ];
arch/powerpc/kvm/e500_mmu.c +1 -1
@@ -783,7 +783,7 @@ int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,if (!pages)return -ENOMEM ; - ret =get_user_pages_fast (cfg->array, num_pages,1 , pages); + ret =get_user_pages_fast (cfg->array, num_pages,FOLL_WRITE , pages);if (ret <0 ) goto free_pages;
arch/powerpc/mm/mmu_context_iommu.c +2 -2
@@ -190,7 +190,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,for (i =0 ; i < entries; ++i) { cur_ua = ua + (i <<PAGE_SHIFT );if (1 !=get_user_pages_fast (cur_ua, -1 /* pages */ ,1 /* iswrite */ , &page)) { +1 /* pages */ ,FOLL_WRITE , &page)) { ret = -EFAULT ;for (j =0 ; j < i; ++j) put_page(pfn_to_page(mem->hpas[j] >>
@@ -209,7 +209,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,if (mm_iommu_move_page_from_cma (page)) goto populate;if (1 !=get_user_pages_fast (cur_ua, -1 /* pages */ ,1 /* iswrite */ , +1 /* pages */ ,FOLL_WRITE , &page)) { ret = -EFAULT ;for (j =0 ; j < i; ++j)
arch/s390/kvm/interrupt.c +1 -1
@@ -2278,7 +2278,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) ret = -EFAULT ; goto out; } - ret =get_user_pages_fast (map->addr,1 ,1 , &map->page); + ret =get_user_pages_fast (map->addr,1 ,FOLL_WRITE , &map->page);if (ret <0 ) goto out;BUG_ON (ret !=1 );
arch/s390/mm/gup.c +6 -6
@@ -265,7 +265,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -277,22 +277,22 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {int nr, ret;might_sleep (); start &=PAGE_MASK ; - nr =__get_user_pages_fast (start, nr_pages, write, pages); + nr =__get_user_pages_fast (start, nr_pages, gup_flags &FOLL_WRITE , + pages);if (nr == nr_pages)return nr;/* Try to get the remaining pages with get_user_pages */ start += nr <<PAGE_SHIFT ; pages += nr; - ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - write ?FOLL_WRITE :0 ); + ret =get_user_pages_unlocked (start, nr_pages - nr, pages, gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) ret = (ret <0 ) ? nr : ret + nr;
arch/sh/mm/gup.c +6 -5
@@ -204,7 +204,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -216,8 +216,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -241,7 +241,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);local_irq_enable ();
@@ -261,7 +262,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/sparc/mm/gup.c +5 -4
@@ -245,8 +245,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,return nr; } -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -303,7 +303,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);
@@ -324,7 +325,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/x86/kvm/paging_tmpl.h +1 -1
@@ -140,7 +140,7 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,pt_element_t *table;struct page *page; - npages =get_user_pages_fast ((unsigned long )ptep_user,1 ,1 , &page); + npages =get_user_pages_fast ((unsigned long )ptep_user,1 ,FOLL_WRITE , &page);/* Check if the user is doing something meaningless. */ if (unlikely (npages !=1 ))return -EFAULT ;
arch/x86/kvm/svm.c +1 -1
@@ -1803,7 +1803,7 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,return NULL ;/* Pin the user virtual address. */ - npinned =get_user_pages_fast (uaddr, npages, write ?FOLL_WRITE :0 , pages); + npinned =get_user_pages_fast (uaddr, npages,FOLL_WRITE , pages);if (npinned != npages) {pr_err ("SEV: Failure locking %lu pages.\n" , npages); goto err;
drivers/fpga/dfl-afu-dma-region.c +1 -1
@@ -102,7 +102,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata, goto unlock_vm; } - pinned =get_user_pages_fast (region->user_addr, npages,1 , + pinned =get_user_pages_fast (region->user_addr, npages,FOLL_WRITE , region->pages);if (pinned <0 ) { ret = pinned;
drivers/gpu/drm/via/via_dmablit.c +2 -1
@@ -239,7 +239,8 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)if (NULL == vsg->pages)return -ENOMEM ; ret =get_user_pages_fast ((unsigned long )xfer->mem_addr, - vsg->num_pages, vsg->direction ==DMA_FROM_DEVICE , + vsg->num_pages, + vsg->direction ==DMA_FROM_DEVICE ?FOLL_WRITE :0 , vsg->pages);if (ret != vsg->num_pages) {if (ret <0 )
drivers/infiniband/hw/hfi1/user_pages.c +2 -1
@@ -105,7 +105,8 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np {int ret; - ret =get_user_pages_fast (vaddr, npages, writable, pages); + ret =get_user_pages_fast (vaddr, npages, writable ?FOLL_WRITE :0 , + pages);if (ret <0 )return ret;
drivers/misc/genwqe/card_utils.c +1 -1
@@ -603,7 +603,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,/* pin user pages in memory */ rc =get_user_pages_fast (data &PAGE_MASK ,/* page aligned addr */ m->nr_pages, - m->write,/* readable/writable */ + m->write ?FOLL_WRITE :0 ,/* readable/writable */ m->page_list);/* ptrs to pages */ if (rc <0 ) goto fail_get_user_pages;
drivers/misc/vmw_vmci/vmci_host.c +1 -1
@@ -242,7 +242,7 @@ static int vmci_host_setup_notify(struct vmci_ctx *context,/* * Lock physical page backing a given user VA. */ - retval =get_user_pages_fast (uva,1 ,1 , &context->notify_page); + retval =get_user_pages_fast (uva,1 ,FOLL_WRITE , &context->notify_page);if (retval !=1 ) { context->notify_page =NULL ;return VMCI_ERROR_GENERIC ;
drivers/misc/vmw_vmci/vmci_queue_pair.c +4 -2
@@ -666,7 +666,8 @@ static int qp_host_get_user_memory(u64 produce_uva,int err =VMCI_SUCCESS ; retval =get_user_pages_fast ((uintptr_t ) produce_uva, - produce_q->kernel_if->num_pages,1 , + produce_q->kernel_if->num_pages, +FOLL_WRITE , produce_q->kernel_if->u.h.header_page);if (retval < (int )produce_q->kernel_if->num_pages) { pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
@@ -678,7 +679,8 @@ static int qp_host_get_user_memory(u64 produce_uva, } retval =get_user_pages_fast ((uintptr_t ) consume_uva, - consume_q->kernel_if->num_pages,1 , + consume_q->kernel_if->num_pages, +FOLL_WRITE , consume_q->kernel_if->u.h.header_page);if (retval < (int )consume_q->kernel_if->num_pages) { pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
drivers/platform/goldfish/goldfish_pipe.c +2 -1
@@ -274,7 +274,8 @@ static int pin_user_pages(unsigned long first_page, *iter_last_page_size = last_page_size; } - ret =get_user_pages_fast (first_page, requested_pages, !is_write, + ret =get_user_pages_fast (first_page, requested_pages, + !is_write ?FOLL_WRITE :0 , pages);if (ret <=0 )return -EFAULT ;
drivers/rapidio/devices/rio_mport_cdev.c +3 -1
@@ -868,7 +868,9 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode, pinned =get_user_pages_fast ( (unsigned long )xfer->loc_addr &PAGE_MASK , - nr_pages, dir ==DMA_FROM_DEVICE , page_list); + nr_pages, + dir ==DMA_FROM_DEVICE ?FOLL_WRITE :0 , + page_list);if (pinned != nr_pages) {if (pinned <0 ) {
drivers/sbus/char/oradax.c +1 -1
@@ -437,7 +437,7 @@ static int dax_lock_page(void *va, struct page **p)dax_dbg ("uva %p" , va); - ret =get_user_pages_fast ((unsigned long )va,1 ,1 , p); + ret =get_user_pages_fast ((unsigned long )va,1 ,FOLL_WRITE , p);if (ret ==1 ) {dax_dbg ("locked page %p, for VA %p" , *p, va);return 0 ;
drivers/scsi/st.c +2 -1
@@ -4918,7 +4918,8 @@ static int sgl_map_user_pages(struct st_buffer *STbp,/* Try to fault in all of the necessary pages */ /* rw==READ means read from drive, write into memory area */ - res =get_user_pages_fast (uaddr, nr_pages, rw ==READ , pages); + res =get_user_pages_fast (uaddr, nr_pages, rw ==READ ?FOLL_WRITE :0 , + pages);/* Errors and no page mapped should return here */ if (res < nr_pages)
drivers/staging/gasket/gasket_page_table.c +2 -2
@@ -486,8 +486,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].dma_addr = pg_tbl->coherent_pages[0 ].paddr + off + i *PAGE_SIZE ; }else { - ret =get_user_pages_fast (page_addr - offset,1 ,1 , - &page); + ret =get_user_pages_fast (page_addr - offset,1 , +FOLL_WRITE , &page);if (ret <=0 ) { dev_err(pg_tbl->device,
drivers/tee/tee_shm.c +1 -1
@@ -273,7 +273,7 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr, goto err; } - rc =get_user_pages_fast (start, num_pages,1 , shm->pages); + rc =get_user_pages_fast (start, num_pages,FOLL_WRITE , shm->pages);if (rc >0 ) shm->num_pages = rc;if (rc != num_pages) {
drivers/vfio/vfio_iommu_spapr_tce.c +2 -1
@@ -532,7 +532,8 @@ static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)enum dma_data_direction direction =iommu_tce_direction (tce);if (get_user_pages_fast (tce &PAGE_MASK ,1 , - direction !=DMA_TO_DEVICE , &page) !=1 ) + direction !=DMA_TO_DEVICE ?FOLL_WRITE :0 , + &page) !=1 )return -EFAULT ; *hpa =__pa ((unsigned long )page_address (page));
drivers/vhost/vhost.c +1 -1
@@ -1700,7 +1700,7 @@ static int set_bit_to_user(int nr, void __user *addr)int bit = nr + (log %PAGE_SIZE ) *8 ;int r; - r =get_user_pages_fast (log,1 ,1 , &page); + r =get_user_pages_fast (log,1 ,FOLL_WRITE , &page);if (r <0 )return r;BUG_ON (r !=1 );
drivers/video/fbdev/pvr2fb.c +1 -1
@@ -686,7 +686,7 @@ static ssize_t pvr2fb_write(struct fb_info *info, const char *buf,if (!pages)return -ENOMEM ; - ret =get_user_pages_fast ((unsigned long )buf, nr_pages, true, pages); + ret =get_user_pages_fast ((unsigned long )buf, nr_pages,FOLL_WRITE , pages);if (ret < nr_pages) { nr_pages = ret; ret = -EINVAL ;
drivers/virt/fsl_hypervisor.c +1 -1
@@ -244,7 +244,7 @@ static long ioctl_memcpy(struct fsl_hv_ioctl_memcpy __user *p)/* Get the physical addresses of the source buffer */ num_pinned =get_user_pages_fast (param.local_vaddr - lb_offset, - num_pages, param.source !=-1 , pages); + num_pages, param.source !=-1 ?FOLL_WRITE :0 , pages);if (num_pinned != num_pages) {/* get_user_pages() failed */
drivers/xen/gntdev.c +1 -1
@@ -852,7 +852,7 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt,unsigned long xen_pfn;int ret; - ret =get_user_pages_fast (addr,1 , writeable, &page); + ret =get_user_pages_fast (addr,1 , writeable ?FOLL_WRITE :0 , &page);if (ret <0 )return ret;
fs/orangefs/orangefs-bufmap.c +1 -1
@@ -269,7 +269,7 @@ orangefs_bufmap_map(struct orangefs_bufmap *bufmap,/* map the pages */ ret =get_user_pages_fast ((unsigned long )user_desc->ptr, - bufmap->page_count,1 , bufmap->page_array); + bufmap->page_count,FOLL_WRITE , bufmap->page_array);if (ret <0 )return ret;
include/linux/mm.h +2 -2
@@ -1537,8 +1537,8 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,long get_user_pages_unlocked (unsigned long start,unsigned long nr_pages,struct page **pages,unsigned int gup_flags); -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages); +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages);/* Container for pinned pfns / pages */ struct frame_vector {
kernel/futex.c +1 -1
@@ -546,7 +546,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, enum futex_aif (unlikely (should_fail_futex (fshared)))return -EFAULT ; - err =get_user_pages_fast (address,1 ,1 , &page); + err =get_user_pages_fast (address,1 ,FOLL_WRITE , &page);/* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access.
lib/iov_iter.c +5 -2
@@ -1280,7 +1280,9 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, len = maxpages *PAGE_SIZE ; addr &= ~(PAGE_SIZE -1 ); n =DIV_ROUND_UP (len,PAGE_SIZE ); - res =get_user_pages_fast (addr, n,iov_iter_rw (i) !=WRITE , pages); + res =get_user_pages_fast (addr, n, +iov_iter_rw (i) !=WRITE ?FOLL_WRITE :0 , + pages);if (unlikely (res <0 ))return res;return (res == n ? len : res *PAGE_SIZE ) - *start;
@@ -1361,7 +1363,8 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, p =get_pages_array (n);if (!p)return -ENOMEM ; - res =get_user_pages_fast (addr, n,iov_iter_rw (i) !=WRITE , p); + res =get_user_pages_fast (addr, n, +iov_iter_rw (i) !=WRITE ?FOLL_WRITE :0 , p);if (unlikely (res <0 )) {kvfree (p);return res;
mm/gup.c +5 -5
@@ -1863,7 +1863,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -1875,8 +1875,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {unsigned long addr, len, end;int nr =0 , ret =0 ;
@@ -1894,7 +1894,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_disable (); -gup_pgd_range (addr, end, write ?FOLL_WRITE :0 , pages, &nr); +gup_pgd_range (addr, end, gup_flags, pages, &nr);local_irq_enable (); ret = nr; }
@@ -1905,7 +1905,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, pages += nr; ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
mm/util.c +4 -4
@@ -306,7 +306,7 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -327,10 +327,10 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);* get_user_pages_fast simply falls back to get_user_pages. */ int __weak get_user_pages_fast (unsigned long start, -int nr_pages,int write,struct page **pages) +int nr_pages,unsigned int gup_flags, +struct page **pages) { -return get_user_pages_unlocked (start, nr_pages, pages, - write ?FOLL_WRITE :0 ); +return get_user_pages_unlocked (start, nr_pages, pages, gup_flags); }EXPORT_SYMBOL_GPL (get_user_pages_fast);
net/ceph/pagevec.c +1 -1
@@ -27,7 +27,7 @@ struct page **ceph_get_direct_page_vector(const void __user *data,while (got < num_pages) { rc =get_user_pages_fast ( (unsigned long )data + ((unsigned long )got *PAGE_SIZE ), - num_pages - got, write_page, pages + got); + num_pages - got, write_page ?FOLL_WRITE :0 , pages + got);if (rc <0 )break ;BUG_ON (rc ==0 );
net/rds/info.c +1 -1
@@ -193,7 +193,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval, ret = -ENOMEM ; goto out; } - ret =get_user_pages_fast (start, nr_pages,1 , pages); + ret =get_user_pages_fast (start, nr_pages,FOLL_WRITE , pages);if (ret != nr_pages) {if (ret >0 ) nr_pages = ret;
net/rds/rdma.c +2 -1
@@ -158,7 +158,8 @@ static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages, {int ret; - ret =get_user_pages_fast (user_addr, nr_pages, write, pages); + ret =get_user_pages_fast (user_addr, nr_pages, write ?FOLL_WRITE :0 , + pages);if (ret >=0 && ret < nr_pages) {while (ret--)
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH V2 3/7] mm/gup: Change GUP fast to use flags rather than a write 'bool'
Quoted 4 lines · On Wed, Feb 13, 2019 at 03:04:51PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
To facilitate additional options to
get_user_pages_fast()change the singular write parameter to begup_flags.
So now we have:
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
struct page **pages, unsigned int gup_flags);
and
int get_user_pages_fast(unsigned long start, int nr_pages,
unsigned int gup_flags, struct page **pages)
Does this make any sense? At least the arguments should be in the same order, I think.
Also this comment:
/*
* get_user_pages_unlocked() is suitable to replace the form:
*
* down_read(&mm->mmap_sem); * get_user_pages(tsk, mm, ..., pages, NULL); * up_read(&mm->mmap_sem);
*
* with:
*
* get_user_pages_unlocked(tsk, mm, ..., pages);
*
* It is functionally equivalent to get_user_pages_fast so
* get_user_pages_fast should be used instead if specific gup_flags
* (e.g. FOLL_FORCE) are not required.
*/
Needs some attention as the recommendation is now nonsense.
Honestly a proper explanation of why two functions exist would be great at this point :)
Jason
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [PATCH V2 3/7] mm/gup: Change GUP fast to use flags rather than a write 'bool'
Quoted 11 lines · Jason Gunthorpe
Quoted 4 lines · On Wed, Feb 13, 2019 at 03:04:51PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
To facilitate additional options to
get_user_pages_fast()change the singular write parameter to begup_flags.So now we have:
long get_user_pages_unlocked(unsigned long start, unsigned long
nr_pages,struct page **pages, unsigned int gup_flags);
Quoted 7 lines
and
int get_user_pages_fast(unsigned long start, int
nr_pages,
unsigned intgup_flags, struct page **pages)Does this make any sense? At least the arguments should be in the same order, I think.
Yes... and no. see below.
Quoted 18 lines
Also this comment:
/*
*get_user_pages_unlocked()is suitable to replace the form:
** down_read(&mm->mmap_sem); * get_user_pages(tsk, mm, ..., pages, NULL); * up_read(&mm->mmap_sem);*
* with:
*
*get_user_pages_unlocked(tsk, mm, ..., pages);
*
* It is functionally equivalent toget_user_pages_fastso
*get_user_pages_fastshould be used instead if specificgup_flags
* (e.g.FOLL_FORCE) are not required.
*/
Quoted 1 line
Needs some attention as the recommendation is now nonsense.
IMO they are not functionally equivalent.
We can't remove *_unlocked() as it is used as both a helper for the arch specific *_fast() calls, _and_ in drivers. Again I don't know the history here but it could be that the drivers should never have used the call in the first place??? Or been converted at some point?
I could change the comment to be something like
/*
* get_user_pages_unlocked() is only to be used by arch specific
* get_user_pages_fast() calls. Drivers should be calling
* get_user_pages_fast()
*/
Instead of the current comment.
And change the drivers to get_user_pages_fast().
However, I'm not sure if these drivers need the FOLL_TOUCH flag which *_unlocked() adds for them. And adding FOLL_TOUCH to *_fast() is not going to give the same functionality.
It _looks_ like we can add FOLL_TOUCH functionality to the fast path in the generic code. I'm not sure about the arch's.
If we did that then we can have those drivers use FOLL_TOUCH or not in *_fast() if they want/need.
Quoted 3 lines
Honestly a proper explanation of why two functions exist would be great at this point :)
I've not researched it. I do agree that there seems to be a lot of calls in this file and the differences are subtle.
Ira
Quoted 2 lines
Jason
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH V2 3/7] mm/gup: Change GUP fast to use flags rather than a write 'bool'
In reply to Jason GunthorpeQuoted 11 lines · Jason Gunthorpe
Quoted 4 lines · On Wed, Feb 13, 2019 at 03:04:51PM -0800, [email protected] wrote:
From: Ira Weiny <[email protected]>
To facilitate additional options to
get_user_pages_fast()change the singular write parameter to begup_flags.So now we have:
long get_user_pages_unlocked(unsigned long start, unsigned long
nr_pages,struct page **pages, unsigned int gup_flags);
Quoted 7 lines
and
int get_user_pages_fast(unsigned long start, int
nr_pages,
unsigned intgup_flags, struct page **pages)Does this make any sense? At least the arguments should be in the same order, I think.
Yes... and no. see below.
Quoted 18 lines
Also this comment:
/*
*get_user_pages_unlocked()is suitable to replace the form:
** down_read(&mm->mmap_sem); * get_user_pages(tsk, mm, ..., pages, NULL); * up_read(&mm->mmap_sem);*
* with:
*
*get_user_pages_unlocked(tsk, mm, ..., pages);
*
* It is functionally equivalent toget_user_pages_fastso
*get_user_pages_fastshould be used instead if specificgup_flags
* (e.g.FOLL_FORCE) are not required.
*/
Quoted 1 line
Needs some attention as the recommendation is now nonsense.
IMO they are not functionally equivalent.
We can't remove *_unlocked() as it is used as both a helper for the arch specific *_fast() calls, _and_ in drivers. Again I don't know the history here but it could be that the drivers should never have used the call in the first place??? Or been converted at some point?
I could change the comment to be something like
/*
* get_user_pages_unlocked() is only to be used by arch specific
* get_user_pages_fast() calls. Drivers should be calling
* get_user_pages_fast()
*/
Instead of the current comment.
And change the drivers to get_user_pages_fast().
However, I'm not sure if these drivers need the FOLL_TOUCH flag which *_unlocked() adds for them. And adding FOLL_TOUCH to *_fast() is not going to give the same functionality.
It _looks_ like we can add FOLL_TOUCH functionality to the fast path in the generic code. I'm not sure about the arch's.
If we did that then we can have those drivers use FOLL_TOUCH or not in *_fast() if they want/need.
Quoted 3 lines
Honestly a proper explanation of why two functions exist would be great at this point :)
I've not researched it. I do agree that there seems to be a lot of calls in this file and the differences are subtle.
Ira
Quoted 2 lines
Jason
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 3/7 mm/gup: Change GUP fast to use flags rather than a write 'bool'
In reply to [email protected]From: Ira Weiny <[email protected]>
To facilitate additional options to get_user_pages_fast() change the singular write parameter to be gup_flags.
This patch does not change any functionality. New functionality will follow in subsequent patches.
Some of the get_user_pages_fast() call sites were unchanged because they already passed FOLL_WRITE or 0 for the write parameter.
--- arch/mips/mm/gup.c | 11 ++++++----- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 ++-- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 ++-- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 ++++++------ arch/sh/mm/gup.c | 11 ++++++----- arch/sparc/mm/gup.c | 9 +++++---- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 ++- drivers/infiniband/hw/hfi1/user_pages.c | 3 ++- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 ++++-- drivers/platform/goldfish/goldfish_pipe.c | 3 ++- drivers/rapidio/devices/rio_mport_cdev.c | 4 +++- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 ++- drivers/staging/gasket/gasket_page_table.c | 4 ++-- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 ++- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 4 ++-- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +++++-- mm/gup.c | 10 +++++----- mm/util.c | 8 ++++---- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 ++- 36 files changed, 81 insertions(+), 65 deletions(-)
arch/mips/mm/gup.c +6 -5
@@ -235,7 +235,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -247,8 +247,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -273,7 +273,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);local_irq_enable ();
@@ -289,7 +290,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, pages += nr; ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , - pages, write ?FOLL_WRITE :0 ); + pages, gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/powerpc/kvm/book3s_64_mmu_hv.c +2 -2
@@ -582,7 +582,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,/* If writing != 0, then the HPTE must allow writing, if we get here */ write_ok = writing; hva =gfn_to_hva_memslot (memslot, gfn); - npages =get_user_pages_fast (hva,1 , writing, pages); + npages =get_user_pages_fast (hva,1 , writing ?FOLL_WRITE :0 , pages);if (npages <1 ) {/* Check if it's an I/O mapping */ down_read (¤t->mm->mmap_sem);
@@ -1175,7 +1175,7 @@ void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,if (!memslot || (memslot->flags &KVM_MEMSLOT_INVALID )) goto err; hva =gfn_to_hva_memslot (memslot, gfn); - npages =get_user_pages_fast (hva,1 ,1 , pages); + npages =get_user_pages_fast (hva,1 ,FOLL_WRITE , pages);if (npages <1 ) goto err; page = pages[0 ];
arch/powerpc/kvm/e500_mmu.c +1 -1
@@ -783,7 +783,7 @@ int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,if (!pages)return -ENOMEM ; - ret =get_user_pages_fast (cfg->array, num_pages,1 , pages); + ret =get_user_pages_fast (cfg->array, num_pages,FOLL_WRITE , pages);if (ret <0 ) goto free_pages;
arch/powerpc/mm/mmu_context_iommu.c +2 -2
@@ -190,7 +190,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,for (i =0 ; i < entries; ++i) { cur_ua = ua + (i <<PAGE_SHIFT );if (1 !=get_user_pages_fast (cur_ua, -1 /* pages */ ,1 /* iswrite */ , &page)) { +1 /* pages */ ,FOLL_WRITE , &page)) { ret = -EFAULT ;for (j =0 ; j < i; ++j) put_page(pfn_to_page(mem->hpas[j] >>
@@ -209,7 +209,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,if (mm_iommu_move_page_from_cma (page)) goto populate;if (1 !=get_user_pages_fast (cur_ua, -1 /* pages */ ,1 /* iswrite */ , +1 /* pages */ ,FOLL_WRITE , &page)) { ret = -EFAULT ;for (j =0 ; j < i; ++j)
arch/s390/kvm/interrupt.c +1 -1
@@ -2278,7 +2278,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) ret = -EFAULT ; goto out; } - ret =get_user_pages_fast (map->addr,1 ,1 , &map->page); + ret =get_user_pages_fast (map->addr,1 ,FOLL_WRITE , &map->page);if (ret <0 ) goto out;BUG_ON (ret !=1 );
arch/s390/mm/gup.c +6 -6
@@ -265,7 +265,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -277,22 +277,22 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {int nr, ret;might_sleep (); start &=PAGE_MASK ; - nr =__get_user_pages_fast (start, nr_pages, write, pages); + nr =__get_user_pages_fast (start, nr_pages, gup_flags &FOLL_WRITE , + pages);if (nr == nr_pages)return nr;/* Try to get the remaining pages with get_user_pages */ start += nr <<PAGE_SHIFT ; pages += nr; - ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - write ?FOLL_WRITE :0 ); + ret =get_user_pages_unlocked (start, nr_pages - nr, pages, gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) ret = (ret <0 ) ? nr : ret + nr;
arch/sh/mm/gup.c +6 -5
@@ -204,7 +204,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -216,8 +216,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -241,7 +241,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);local_irq_enable ();
@@ -261,7 +262,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/sparc/mm/gup.c +5 -4
@@ -245,8 +245,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,return nr; } -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {struct mm_struct *mm = current->mm;unsigned long addr, len, end;
@@ -303,7 +303,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, next =pgd_addr_end (addr, end);if (pgd_none (pgd)) goto slow; -if (!gup_pud_range (pgd, addr, next, write, pages, &nr)) +if (!gup_pud_range (pgd, addr, next, gup_flags &FOLL_WRITE , + pages, &nr)) goto slow; }while (pgdp++, addr = next, addr != end);
@@ -324,7 +325,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, ret =get_user_pages_unlocked (start, (end - start) >>PAGE_SHIFT , pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
arch/x86/kvm/paging_tmpl.h +1 -1
@@ -140,7 +140,7 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,pt_element_t *table;struct page *page; - npages =get_user_pages_fast ((unsigned long )ptep_user,1 ,1 , &page); + npages =get_user_pages_fast ((unsigned long )ptep_user,1 ,FOLL_WRITE , &page);/* Check if the user is doing something meaningless. */ if (unlikely (npages !=1 ))return -EFAULT ;
arch/x86/kvm/svm.c +1 -1
@@ -1803,7 +1803,7 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,return NULL ;/* Pin the user virtual address. */ - npinned =get_user_pages_fast (uaddr, npages, write ?FOLL_WRITE :0 , pages); + npinned =get_user_pages_fast (uaddr, npages,FOLL_WRITE , pages);if (npinned != npages) {pr_err ("SEV: Failure locking %lu pages.\n" , npages); goto err;
drivers/fpga/dfl-afu-dma-region.c +1 -1
@@ -102,7 +102,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata, goto unlock_vm; } - pinned =get_user_pages_fast (region->user_addr, npages,1 , + pinned =get_user_pages_fast (region->user_addr, npages,FOLL_WRITE , region->pages);if (pinned <0 ) { ret = pinned;
drivers/gpu/drm/via/via_dmablit.c +2 -1
@@ -239,7 +239,8 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)if (NULL == vsg->pages)return -ENOMEM ; ret =get_user_pages_fast ((unsigned long )xfer->mem_addr, - vsg->num_pages, vsg->direction ==DMA_FROM_DEVICE , + vsg->num_pages, + vsg->direction ==DMA_FROM_DEVICE ?FOLL_WRITE :0 , vsg->pages);if (ret != vsg->num_pages) {if (ret <0 )
drivers/infiniband/hw/hfi1/user_pages.c +2 -1
@@ -105,7 +105,8 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np {int ret; - ret =get_user_pages_fast (vaddr, npages, writable, pages); + ret =get_user_pages_fast (vaddr, npages, writable ?FOLL_WRITE :0 , + pages);if (ret <0 )return ret;
drivers/misc/genwqe/card_utils.c +1 -1
@@ -603,7 +603,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,/* pin user pages in memory */ rc =get_user_pages_fast (data &PAGE_MASK ,/* page aligned addr */ m->nr_pages, - m->write,/* readable/writable */ + m->write ?FOLL_WRITE :0 ,/* readable/writable */ m->page_list);/* ptrs to pages */ if (rc <0 ) goto fail_get_user_pages;
drivers/misc/vmw_vmci/vmci_host.c +1 -1
@@ -242,7 +242,7 @@ static int vmci_host_setup_notify(struct vmci_ctx *context,/* * Lock physical page backing a given user VA. */ - retval =get_user_pages_fast (uva,1 ,1 , &context->notify_page); + retval =get_user_pages_fast (uva,1 ,FOLL_WRITE , &context->notify_page);if (retval !=1 ) { context->notify_page =NULL ;return VMCI_ERROR_GENERIC ;
drivers/misc/vmw_vmci/vmci_queue_pair.c +4 -2
@@ -666,7 +666,8 @@ static int qp_host_get_user_memory(u64 produce_uva,int err =VMCI_SUCCESS ; retval =get_user_pages_fast ((uintptr_t ) produce_uva, - produce_q->kernel_if->num_pages,1 , + produce_q->kernel_if->num_pages, +FOLL_WRITE , produce_q->kernel_if->u.h.header_page);if (retval < (int )produce_q->kernel_if->num_pages) { pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
@@ -678,7 +679,8 @@ static int qp_host_get_user_memory(u64 produce_uva, } retval =get_user_pages_fast ((uintptr_t ) consume_uva, - consume_q->kernel_if->num_pages,1 , + consume_q->kernel_if->num_pages, +FOLL_WRITE , consume_q->kernel_if->u.h.header_page);if (retval < (int )consume_q->kernel_if->num_pages) { pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
drivers/platform/goldfish/goldfish_pipe.c +2 -1
@@ -274,7 +274,8 @@ static int pin_user_pages(unsigned long first_page, *iter_last_page_size = last_page_size; } - ret =get_user_pages_fast (first_page, requested_pages, !is_write, + ret =get_user_pages_fast (first_page, requested_pages, + !is_write ?FOLL_WRITE :0 , pages);if (ret <=0 )return -EFAULT ;
drivers/rapidio/devices/rio_mport_cdev.c +3 -1
@@ -868,7 +868,9 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode, pinned =get_user_pages_fast ( (unsigned long )xfer->loc_addr &PAGE_MASK , - nr_pages, dir ==DMA_FROM_DEVICE , page_list); + nr_pages, + dir ==DMA_FROM_DEVICE ?FOLL_WRITE :0 , + page_list);if (pinned != nr_pages) {if (pinned <0 ) {
drivers/sbus/char/oradax.c +1 -1
@@ -437,7 +437,7 @@ static int dax_lock_page(void *va, struct page **p)dax_dbg ("uva %p" , va); - ret =get_user_pages_fast ((unsigned long )va,1 ,1 , p); + ret =get_user_pages_fast ((unsigned long )va,1 ,FOLL_WRITE , p);if (ret ==1 ) {dax_dbg ("locked page %p, for VA %p" , *p, va);return 0 ;
drivers/scsi/st.c +2 -1
@@ -4918,7 +4918,8 @@ static int sgl_map_user_pages(struct st_buffer *STbp,/* Try to fault in all of the necessary pages */ /* rw==READ means read from drive, write into memory area */ - res =get_user_pages_fast (uaddr, nr_pages, rw ==READ , pages); + res =get_user_pages_fast (uaddr, nr_pages, rw ==READ ?FOLL_WRITE :0 , + pages);/* Errors and no page mapped should return here */ if (res < nr_pages)
drivers/staging/gasket/gasket_page_table.c +2 -2
@@ -486,8 +486,8 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].dma_addr = pg_tbl->coherent_pages[0 ].paddr + off + i *PAGE_SIZE ; }else { - ret =get_user_pages_fast (page_addr - offset,1 ,1 , - &page); + ret =get_user_pages_fast (page_addr - offset,1 , +FOLL_WRITE , &page);if (ret <=0 ) { dev_err(pg_tbl->device,
drivers/tee/tee_shm.c +1 -1
@@ -273,7 +273,7 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr, goto err; } - rc =get_user_pages_fast (start, num_pages,1 , shm->pages); + rc =get_user_pages_fast (start, num_pages,FOLL_WRITE , shm->pages);if (rc >0 ) shm->num_pages = rc;if (rc != num_pages) {
drivers/vfio/vfio_iommu_spapr_tce.c +2 -1
@@ -532,7 +532,8 @@ static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)enum dma_data_direction direction =iommu_tce_direction (tce);if (get_user_pages_fast (tce &PAGE_MASK ,1 , - direction !=DMA_TO_DEVICE , &page) !=1 ) + direction !=DMA_TO_DEVICE ?FOLL_WRITE :0 , + &page) !=1 )return -EFAULT ; *hpa =__pa ((unsigned long )page_address (page));
drivers/vhost/vhost.c +1 -1
@@ -1700,7 +1700,7 @@ static int set_bit_to_user(int nr, void __user *addr)int bit = nr + (log %PAGE_SIZE ) *8 ;int r; - r =get_user_pages_fast (log,1 ,1 , &page); + r =get_user_pages_fast (log,1 ,FOLL_WRITE , &page);if (r <0 )return r;BUG_ON (r !=1 );
drivers/video/fbdev/pvr2fb.c +1 -1
@@ -686,7 +686,7 @@ static ssize_t pvr2fb_write(struct fb_info *info, const char *buf,if (!pages)return -ENOMEM ; - ret =get_user_pages_fast ((unsigned long )buf, nr_pages, true, pages); + ret =get_user_pages_fast ((unsigned long )buf, nr_pages,FOLL_WRITE , pages);if (ret < nr_pages) { nr_pages = ret; ret = -EINVAL ;
drivers/virt/fsl_hypervisor.c +1 -1
@@ -244,7 +244,7 @@ static long ioctl_memcpy(struct fsl_hv_ioctl_memcpy __user *p)/* Get the physical addresses of the source buffer */ num_pinned =get_user_pages_fast (param.local_vaddr - lb_offset, - num_pages, param.source !=-1 , pages); + num_pages, param.source !=-1 ?FOLL_WRITE :0 , pages);if (num_pinned != num_pages) {/* get_user_pages() failed */
drivers/xen/gntdev.c +1 -1
@@ -852,7 +852,7 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt,unsigned long xen_pfn;int ret; - ret =get_user_pages_fast (addr,1 , writeable, &page); + ret =get_user_pages_fast (addr,1 , writeable ?FOLL_WRITE :0 , &page);if (ret <0 )return ret;
fs/orangefs/orangefs-bufmap.c +1 -1
@@ -269,7 +269,7 @@ orangefs_bufmap_map(struct orangefs_bufmap *bufmap,/* map the pages */ ret =get_user_pages_fast ((unsigned long )user_desc->ptr, - bufmap->page_count,1 , bufmap->page_array); + bufmap->page_count,FOLL_WRITE , bufmap->page_array);if (ret <0 )return ret;
include/linux/mm.h +2 -2
@@ -1537,8 +1537,8 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,long get_user_pages_unlocked (unsigned long start,unsigned long nr_pages,struct page **pages,unsigned int gup_flags); -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages); +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages);/* Container for pinned pfns / pages */ struct frame_vector {
kernel/futex.c +1 -1
@@ -546,7 +546,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, enum futex_aif (unlikely (should_fail_futex (fshared)))return -EFAULT ; - err =get_user_pages_fast (address,1 ,1 , &page); + err =get_user_pages_fast (address,1 ,FOLL_WRITE , &page);/* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access.
lib/iov_iter.c +5 -2
@@ -1280,7 +1280,9 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, len = maxpages *PAGE_SIZE ; addr &= ~(PAGE_SIZE -1 ); n =DIV_ROUND_UP (len,PAGE_SIZE ); - res =get_user_pages_fast (addr, n,iov_iter_rw (i) !=WRITE , pages); + res =get_user_pages_fast (addr, n, +iov_iter_rw (i) !=WRITE ?FOLL_WRITE :0 , + pages);if (unlikely (res <0 ))return res;return (res == n ? len : res *PAGE_SIZE ) - *start;
@@ -1361,7 +1363,8 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, p =get_pages_array (n);if (!p)return -ENOMEM ; - res =get_user_pages_fast (addr, n,iov_iter_rw (i) !=WRITE , p); + res =get_user_pages_fast (addr, n, +iov_iter_rw (i) !=WRITE ?FOLL_WRITE :0 , p);if (unlikely (res <0 )) {kvfree (p);return res;
mm/gup.c +5 -5
@@ -1863,7 +1863,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -1875,8 +1875,8 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,* requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. */ -int get_user_pages_fast (unsigned long start,int nr_pages,int write, -struct page **pages) +int get_user_pages_fast (unsigned long start,int nr_pages, +unsigned int gup_flags,struct page **pages) {unsigned long addr, len, end;int nr =0 , ret =0 ;
@@ -1894,7 +1894,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,if (gup_fast_permitted (start, nr_pages)) {local_irq_disable (); -gup_pgd_range (addr, end, write ?FOLL_WRITE :0 , pages, &nr); +gup_pgd_range (addr, end, gup_flags, pages, &nr);local_irq_enable (); ret = nr; }
@@ -1905,7 +1905,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, pages += nr; ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - write ?FOLL_WRITE :0 ); + gup_flags);/* Have to be a bit careful with return values */ if (nr >0 ) {
mm/util.c +4 -4
@@ -306,7 +306,7 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);* get_user_pages_fast() - pin user pages in memory * @start: starting user address * @nr_pages: number of pages from start to pin -* @write: whether pages will be written to +* @gup_flags: flags modifying pin behaviour * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. *
@@ -327,10 +327,10 @@ EXPORT_SYMBOL_GPL(__get_user_pages_fast);* get_user_pages_fast simply falls back to get_user_pages. */ int __weak get_user_pages_fast (unsigned long start, -int nr_pages,int write,struct page **pages) +int nr_pages,unsigned int gup_flags, +struct page **pages) { -return get_user_pages_unlocked (start, nr_pages, pages, - write ?FOLL_WRITE :0 ); +return get_user_pages_unlocked (start, nr_pages, pages, gup_flags); }EXPORT_SYMBOL_GPL (get_user_pages_fast);
net/ceph/pagevec.c +1 -1
@@ -27,7 +27,7 @@ struct page **ceph_get_direct_page_vector(const void __user *data,while (got < num_pages) { rc =get_user_pages_fast ( (unsigned long )data + ((unsigned long )got *PAGE_SIZE ), - num_pages - got, write_page, pages + got); + num_pages - got, write_page ?FOLL_WRITE :0 , pages + got);if (rc <0 )break ;BUG_ON (rc ==0 );
net/rds/info.c +1 -1
@@ -193,7 +193,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval, ret = -ENOMEM ; goto out; } - ret =get_user_pages_fast (start, nr_pages,1 , pages); + ret =get_user_pages_fast (start, nr_pages,FOLL_WRITE , pages);if (ret != nr_pages) {if (ret >0 ) nr_pages = ret;
net/rds/rdma.c +2 -1
@@ -158,7 +158,8 @@ static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages, {int ret; - ret =get_user_pages_fast (user_addr, nr_pages, write, pages); + ret =get_user_pages_fast (user_addr, nr_pages, write ?FOLL_WRITE :0 , + pages);if (ret >=0 && ret < nr_pages) {while (ret--)
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 4/7 mm/gup: Add FOLL_LONGTERM capability to GUP fast
In reply to [email protected]From: Ira Weiny <[email protected]>
DAX pages were previously unprotected from longterm pins when users called get_user_pages_fast().
Use the new FOLL_LONGTERM flag to check for DEVMAP pages and fall back to regular GUP processing if a DEVMAP page is encountered.
--- mm/gup.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-)
mm/gup.c +21 -3
@@ -1439,6 +1439,9 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, goto pte_unmap;if (pte_devmap (pte)) { +if (unlikely (flags &FOLL_LONGTERM )) + goto pte_unmap; + pgmap =get_dev_pagemap (pte_pfn (pte), pgmap);if (unlikely (!pgmap)) {undo_dev_pagemap (nr, nr_start, pages);
@@ -1578,8 +1581,11 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,if (!pmd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ; -if (pmd_devmap (orig)) +if (pmd_devmap (orig)) { +if (unlikely (flags &FOLL_LONGTERM )) +return 0 ;return __gup_device_huge_pmd (orig, pmdp, addr, end, pages, nr); + } refs =0 ; page =pmd_page (orig) + ((addr & ~PMD_MASK ) >>PAGE_SHIFT );
@@ -1904,8 +1910,20 @@ int get_user_pages_fast(unsigned long start, int nr_pages, start += nr <<PAGE_SHIFT ; pages += nr; - ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - gup_flags); +if (gup_flags &FOLL_LONGTERM ) { +down_read (¤t->mm->mmap_sem); + ret =__gup_longterm_locked (current, current->mm, + start, nr_pages - nr, + pages,NULL , gup_flags); +up_read (¤t->mm->mmap_sem); + }else { +/* +* retain FAULT_FOLL_ALLOW_RETRY optimization if +* possible +*/ + ret =get_user_pages_unlocked (start, nr_pages - nr, + pages, gup_flags); + }/* Have to be a bit careful with return values */ if (nr >0 ) {
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 4/7 mm/gup: Add FOLL_LONGTERM capability to GUP fast
In reply to [email protected]From: Ira Weiny <[email protected]>
DAX pages were previously unprotected from longterm pins when users called get_user_pages_fast().
Use the new FOLL_LONGTERM flag to check for DEVMAP pages and fall back to regular GUP processing if a DEVMAP page is encountered.
--- mm/gup.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-)
mm/gup.c +21 -3
@@ -1439,6 +1439,9 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, goto pte_unmap;if (pte_devmap (pte)) { +if (unlikely (flags &FOLL_LONGTERM )) + goto pte_unmap; + pgmap =get_dev_pagemap (pte_pfn (pte), pgmap);if (unlikely (!pgmap)) {undo_dev_pagemap (nr, nr_start, pages);
@@ -1578,8 +1581,11 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,if (!pmd_access_permitted (orig, flags &FOLL_WRITE ))return 0 ; -if (pmd_devmap (orig)) +if (pmd_devmap (orig)) { +if (unlikely (flags &FOLL_LONGTERM )) +return 0 ;return __gup_device_huge_pmd (orig, pmdp, addr, end, pages, nr); + } refs =0 ; page =pmd_page (orig) + ((addr & ~PMD_MASK ) >>PAGE_SHIFT );
@@ -1904,8 +1910,20 @@ int get_user_pages_fast(unsigned long start, int nr_pages, start += nr <<PAGE_SHIFT ; pages += nr; - ret =get_user_pages_unlocked (start, nr_pages - nr, pages, - gup_flags); +if (gup_flags &FOLL_LONGTERM ) { +down_read (¤t->mm->mmap_sem); + ret =__gup_longterm_locked (current, current->mm, + start, nr_pages - nr, + pages,NULL , gup_flags); +up_read (¤t->mm->mmap_sem); + }else { +/* +* retain FAULT_FOLL_ALLOW_RETRY optimization if +* possible +*/ + ret =get_user_pages_unlocked (start, nr_pages - nr, + pages, gup_flags); + }/* Have to be a bit careful with return values */ if (nr >0 ) {
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 5/7 IB/hfi1: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/hfi1/user_pages.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hfi1/user_pages.c +4 -2
@@ -104,9 +104,11 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np bool writable, struct page **pages) { int ret; +unsigned int gup_flags = writable ?FOLL_WRITE :0 ; - ret =get_user_pages_fast (vaddr, npages, writable ?FOLL_WRITE :0 , - pages); + gup_flags |=FOLL_LONGTERM ; + + ret =get_user_pages_fast (vaddr, npages, gup_flags, pages);if (ret <0 )return ret;
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 5/7 IB/hfi1: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/hfi1/user_pages.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hfi1/user_pages.c +4 -2
@@ -104,9 +104,11 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np bool writable, struct page **pages) { int ret; +unsigned int gup_flags = writable ?FOLL_WRITE :0 ; - ret =get_user_pages_fast (vaddr, npages, writable ?FOLL_WRITE :0 , - pages); + gup_flags |=FOLL_LONGTERM ; + + ret =get_user_pages_fast (vaddr, npages, gup_flags, pages);if (ret <0 )return ret;
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 6/7 IB/qib: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/hw/qib/qib_user_sdma.c +1 -1
@@ -673,7 +673,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,else j = npages; - ret =get_user_pages_fast (addr, j,0 , pages); + ret =get_user_pages_fast (addr, j,FOLL_LONGTERM , pages);if (ret != j) { i =0 ; j = ret;
Signature · 6 lines
2.20.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 6/7 IB/qib: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/hw/qib/qib_user_sdma.c +1 -1
@@ -673,7 +673,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,else j = npages; - ret =get_user_pages_fast (addr, j,0 , pages); + ret =get_user_pages_fast (addr, j,FOLL_LONGTERM , pages);if (ret != j) { i =0 ; j = ret;
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH v2 7/7 IB/mthca: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/mthca/mthca_memfree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mthca/mthca_memfree.c +2 -1
@@ -472,7 +472,8 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, goto out; } - ret =get_user_pages_fast (uaddr &PAGE_MASK ,1 ,FOLL_WRITE , pages); + ret =get_user_pages_fast (uaddr &PAGE_MASK ,1 , +FOLL_WRITE |FOLL_LONGTERM , pages);if (ret <0 ) goto out;
Signature · 1 line
2.20.1
PATCH v2 7/7 IB/mthca: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
In reply to [email protected]From: Ira Weiny <[email protected]>
Use the new FOLL_LONGTERM to get_user_pages_fast() to protect against FS DAX pages being mapped.
--- drivers/infiniband/hw/mthca/mthca_memfree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/infiniband/hw/mthca/mthca_memfree.c +2 -1
@@ -472,7 +472,8 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, goto out; } - ret =get_user_pages_fast (uaddr &PAGE_MASK ,1 ,FOLL_WRITE , pages); + ret =get_user_pages_fast (uaddr &PAGE_MASK ,1 , +FOLL_WRITE |FOLL_LONGTERM , pages);if (ret <0 ) goto out;
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
v2 cover Add FOLL_LONGTERM to GUP fast and use it
In reply to [email protected]From: Ira Weiny <[email protected]>
NOTE: This series depends on my clean up patch to remove the write parameter from gup_fast_permitted()[1]
HFI1, qib, and mthca, use get_user_pages_fast() due to it performance advantages. These pages can be held for a significant time. But get_user_pages_fast() does not protect against mapping of FS DAX pages.
Introduce FOLL_LONGTERM and use this flag in get_user_pages_fast() which retains the performance while also adding the FS DAX checks. XDP has also shown interest in using this functionality.[2]
In addition we change get_user_pages() to use the new FOLL_LONGTERM flag and remove the specialized get_user_pages_longterm call.
[1] https://lkml.org/lkml/2019/2/11/237
[2] https://lkml.org/lkml/2019/2/11/1789
Ira Weiny (7):
mm/gup: Replace get_user_pages_longterm() with FOLL_LONGTERM mm/gup: Change write parameter to flags in fast walk mm/gup: Change GUP fast to use flags rather than a write 'bool' mm/gup: Add FOLL_LONGTERM capability to GUP fast IB/hfi1: Use the new FOLL_LONGTERM flag to get_user_pages_fast() IB/qib: Use the new FOLL_LONGTERM flag to get_user_pages_fast() IB/mthca: Use the new FOLL_LONGTERM flag to get_user_pages_fast()
arch/mips/mm/gup.c | 11 +- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 +- arch/powerpc/kvm/e500_mmu.c | 2 +- arch/powerpc/mm/mmu_context_iommu.c | 4 +- arch/s390/kvm/interrupt.c | 2 +- arch/s390/mm/gup.c | 12 +- arch/sh/mm/gup.c | 11 +- arch/sparc/mm/gup.c | 9 +- arch/x86/kvm/paging_tmpl.h | 2 +- arch/x86/kvm/svm.c | 2 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/gpu/drm/via/via_dmablit.c | 3 +- drivers/infiniband/core/umem.c | 5 +- drivers/infiniband/hw/hfi1/user_pages.c | 5 +- drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +- drivers/infiniband/hw/qib/qib_user_pages.c | 8 +- drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +- drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 6 +- drivers/misc/genwqe/card_utils.c | 2 +- drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/misc/vmw_vmci/vmci_queue_pair.c | 6 +- drivers/platform/goldfish/goldfish_pipe.c | 3 +- drivers/rapidio/devices/rio_mport_cdev.c | 4 +- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/st.c | 3 +- drivers/staging/gasket/gasket_page_table.c | 4 +- drivers/tee/tee_shm.c | 2 +- drivers/vfio/vfio_iommu_spapr_tce.c | 3 +- drivers/vfio/vfio_iommu_type1.c | 3 +- drivers/vhost/vhost.c | 2 +- drivers/video/fbdev/pvr2fb.c | 2 +- drivers/virt/fsl_hypervisor.c | 2 +- drivers/xen/gntdev.c | 2 +- fs/orangefs/orangefs-bufmap.c | 2 +- include/linux/mm.h | 17 +- kernel/futex.c | 2 +- lib/iov_iter.c | 7 +- mm/gup.c | 220 ++++++++++++-------- mm/gup_benchmark.c | 5 +- mm/util.c | 8 +- net/ceph/pagevec.c | 2 +- net/rds/info.c | 2 +- net/rds/rdma.c | 3 +- 44 files changed, 232 insertions(+), 180 deletions(-)
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel