mm: add account_locked_vm utility function
Subsystems (9)
PATCH 1/1 mm: add account_locked_vm utility function
locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly. Error codes stay the same, so user-visible behavior does too.
---
Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.
Andrew, this one patch replaces these six from [1]:
mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch
That series converts locked_vm to an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers use pinned_vm instead.
Christophe's suggestion of cmpxchg[2] does prevent the races he mentioned for pinned_vm, but others would still remain. In perf_mmap and the hfi1 driver, pinned_vm is checked against the rlimit racily and then later increased when the pinned_vm originally read may have gone stale. Any fixes for that, that I could think of, seem about as good as what's there now, so I left it. I have a patch that uses cmpxchg with pinned_vm if others feel strongly that the aforementioned races should be fixed.
Daniel
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/
arch/powerpc/kvm/book3s_64_vio.c | 44 +++--------------------- arch/powerpc/mm/mmu_context_iommu.c | 41 +++------------------- drivers/fpga/dfl-afu-dma-region.c | 53 +++-------------------------- drivers/vfio/vfio_iommu_spapr_tce.c | 52 +++++----------------------- drivers/vfio/vfio_iommu_type1.c | 23 ++++--------- include/linux/mm.h | 19 +++++++++++ mm/util.c | 48 ++++++++++++++++++++++++++ 7 files changed, 94 insertions(+), 186 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
@@ -30,6 +30,7 @@#include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)return tce_pages +ALIGN (stt_bytes,PAGE_SIZE ) /PAGE_SIZE ; } -static long kvmppc_account_memlimit (unsigned long stt_pages,bool inc) -{ -long ret =0 ; - -if (!current || !current->mm) -return ret;/* process exited */ - -down_write (¤t->mm->mmap_sem); - -if (inc) { -unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += stt_pages; - }else { -if (WARN_ON_ONCE (stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n" , current->pid, - inc ?'+' :'-' , - stt_pages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -static void kvm_spapr_tce_iommu_table_free (struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -277,7 +241,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)kvm_put_kvm (stt->kvm); -kvmppc_account_memlimit ( +account_locked_vm (current->mm,kvmppc_stt_pages (kvmppc_tce_pages (stt->size)), false);call_rcu (&stt->rcu, release_spapr_tce_table);
@@ -303,7 +267,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,return -EINVAL ; npages =kvmppc_tce_pages (size); - ret =kvmppc_account_memlimit (kvmppc_stt_pages (npages), true); + ret =account_locked_vm (current->mm,kvmppc_stt_pages (npages), true);if (ret)return ret;
@@ -359,7 +323,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,kfree (stt); fail_acct: -kvmppc_account_memlimit (kvmppc_stt_pages (npages), false); +account_locked_vm (current->mm,kvmppc_stt_pages (npages), false);return ret; }
arch/powerpc/mm/mmu_context_iommu.c +4 -37
71 lines not on this page · raw message
drivers/fpga/dfl-afu-dma-region.c +4 -49
83 lines not on this page · raw message
drivers/vfio/vfio_iommu_spapr_tce.c +9 -43
92 lines not on this page · raw message
drivers/vfio/vfio_iommu_type1.c +6 -17
@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)return -ESRCH ;/* process exited */ ret =down_write_killable (&mm->mmap_sem); -if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit) - ret = -ENOMEM ; - } - } +if (ret) + goto out; -if (!ret) - mm->locked_vm += npage; - -up_write (&mm->mmap_sem); - } + ret =__account_locked_vm (mm,abs (npage), npage >0 , dma->task, + dma->lock_cap); +up_write (&mm->mmap_sem); +out:if (async)mmput (mm);
include/linux/mm.h +19 -0
@@ -1521,6 +1521,25 @@ static inline long get_user_pages_longterm(unsigned long start,int get_user_pages_fast (unsigned long start,int nr_pages,int write,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */
mm/util.c +48 -0
60 lines not on this page · raw message
base-commit: 37624b58542fb9f2d9a70e6ea006ef8a5f66c30b
Signature · 1 line
2.21.0
Quoted 39 lines · Daniel Jordan
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly. Error codes stay the same, so user-visible behavior does too.Signed-off-by Daniel Jordan <[email protected]>Cc Alan Tull <[email protected]>Cc Alexey Kardashevskiy <[email protected]>Cc Alex Williamson <[email protected]>Cc Andrew Morton <[email protected]>Cc Benjamin Herrenschmidt <[email protected]>Cc Christoph Lameter <[email protected]>Cc Christophe Leroy <[email protected]>Cc Davidlohr Bueso <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc Mark Rutland <[email protected]>Cc Michael Ellerman <[email protected]>Cc Moritz Fischer <[email protected]>Cc Paul Mackerras <[email protected]>Cc Steve Sistare <[email protected]>Cc Wu Hao <[email protected]>Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.
Andrew, this one patch replaces these six from [1]:
mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch
Quoted 4 lines
That series converts
locked_vmto an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers usepinned_vminstead.
Did we ever decide what to do here? Should all these drivers be switched to pinned_vm anyhow?
Jason
Quoted 10 lines · Jason Gunthorpe
Quoted 9 lines · Daniel Jordan
Andrew, this one patch replaces these six from [1]:
mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch
Quoted 7 lines
Quoted 4 lines
That series converts
locked_vmto an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers usepinned_vminstead.Did we ever decide what to do here? Should all these drivers be switched to
pinned_vmanyhow?
Well, there were the concerns about switching in [1]. Alex, is there an example of an application or library that would break or be exploitable? If there were particular worries (qemu for vfio type1, for example), perhaps some coordinated changes across the kernel and userspace would be possible, especially given the amount of effort it's likely going to take to get the locked_vm/pinned_vm accounting sorted out.
Quoted 3 lines · Daniel Jordan
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.
And I rather liked that prints were different and tell precisely which one of three each printk is.
I commented below but in general this seems working.
Quoted 335 lines
Error codes stay the same, so user-visible behavior does too.
Signed-off-by Daniel Jordan <[email protected]>Cc Alan Tull <[email protected]>Cc Alexey Kardashevskiy <[email protected]>Cc Alex Williamson <[email protected]>Cc Andrew Morton <[email protected]>Cc Benjamin Herrenschmidt <[email protected]>Cc Christoph Lameter <[email protected]>Cc Christophe Leroy <[email protected]>Cc Davidlohr Bueso <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc Mark Rutland <[email protected]>Cc Michael Ellerman <[email protected]>Cc Moritz Fischer <[email protected]>Cc Paul Mackerras <[email protected]>Cc Steve Sistare <[email protected]>Cc Wu Hao <[email protected]>---
Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.
Andrew, this one patch replaces these six from [1]:
mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch
That series converts
locked_vmto an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers usepinned_vminstead.Christophe's suggestion of cmpxchg[2] does prevent the races he mentioned for
pinned_vm, but others would still remain. Inperf_mmapand the hfi1 driver,pinned_vmis checked against the rlimit racily and then later increased when thepinned_vmoriginally read may have gone stale. Any fixes for that, that I could think of, seem about as good as what's there now, so I left it. I have a patch that uses cmpxchg withpinned_vmif others feel strongly that the aforementioned races should be fixed.Daniel
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/arch/powerpc/kvm/book3s_64_vio.c | 44 +++--------------------- arch/powerpc/mm/mmu_context_iommu.c | 41 +++------------------- drivers/fpga/dfl-afu-dma-region.c | 53 +++-------------------------- drivers/vfio/vfio_iommu_spapr_tce.c | 52 +++++----------------------- drivers/vfio/vfio_iommu_type1.c | 23 ++++--------- include/linux/mm.h | 19 +++++++++++ mm/util.c | 48 ++++++++++++++++++++++++++ 7 files changed, 94 insertions(+), 186 deletions(-)arch/powerpc/kvm/book3s_64_vio.c +4
-40@@ -30,6 +30,7 @@#include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> @@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)return tce_pages +ALIGN (stt_bytes,PAGE_SIZE ) /PAGE_SIZE ; } -static long kvmppc_account_memlimit (unsigned long stt_pages,bool inc) -{ -long ret =0 ; - -if (!current || !current->mm) -return ret;/* process exited */ - -down_write (¤t->mm->mmap_sem); - -if (inc) { -unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += stt_pages; - }else { -if (WARN_ON_ONCE (stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n" , current->pid, - inc ?'+' :'-' , - stt_pages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -static void kvm_spapr_tce_iommu_table_free (struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,@@ -277,7 +241,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)kvm_put_kvm (stt->kvm); -kvmppc_account_memlimit ( +account_locked_vm (current->mm,kvmppc_stt_pages (kvmppc_tce_pages (stt->size)), false);call_rcu (&stt->rcu, release_spapr_tce_table);@@ -303,7 +267,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,return -EINVAL ; npages =kvmppc_tce_pages (size); - ret =kvmppc_account_memlimit (kvmppc_stt_pages (npages), true); + ret =account_locked_vm (current->mm,kvmppc_stt_pages (npages), true);if (ret)return ret;@@ -359,7 +323,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,kfree (stt); fail_acct: -kvmppc_account_memlimit (kvmppc_stt_pages (npages), false); +account_locked_vm (current->mm,kvmppc_stt_pages (npages), false);return ret; }arch/powerpc/mm/mmu_context_iommu.c +4
-37@@ -19,6 +19,7 @@#include <linux/hugetlb.h> #include <linux/swap.h> #include <linux/sizes.h> +#include <linux/mm.h> #include <asm/mmu_context.h> #include <asm/pte-walk.h> #include <linux/mm_inline.h> @@ -51,40 +52,6 @@ struct mm_iommu_table_group_mem_t {u64 dev_hpa;/* Device memory base address */ }; -static long mm_iommu_adjust_locked_vm (struct mm_struct *mm, -unsigned long npages,bool incr) -{ -long ret =0 , locked, lock_limit; - -if (!npages) -return 0 ; - -down_write (&mm->mmap_sem); - -if (incr) { - locked = mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - mm->locked_vm += npages; - }else { -if (WARN_ON_ONCE (npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n" , - current ? current->pid :0 , - incr ?'+' :'-' , - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK )); -up_write (&mm->mmap_sem); - -return ret; -} -bool mm_iommu_preregistered (struct mm_struct *mm) {return !list_empty (&mm->context.iommu_group_mem_list);@@ -101,7 +68,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,unsigned long entry, chunk;if (dev_hpa =MM_IOMMU_TABLE_INVALID_HPA ) { - ret =mm_iommu_adjust_locked_vm (mm, entries, true); + ret =account_locked_vm (mm, entries, true);if (ret)return ret;@@ -215,7 +182,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,kfree (mem); unlock_exit: -mm_iommu_adjust_locked_vm (mm, locked_entries, false); +account_locked_vm (mm, locked_entries, false);return ret; }@@ -315,7 +282,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) unlock_exit:mutex_unlock (&mem_list_mutex); -mm_iommu_adjust_locked_vm (mm, unlock_entries, false); +account_locked_vm (mm, unlock_entries, false);return ret; }drivers/fpga/dfl-afu-dma-region.c +4
-49@@ -12,6 +12,7 @@#include <linux/dma-mapping.h> #include <linux/sched/signal.h> #include <linux/uaccess.h> +#include <linux/mm.h> #include "dfl-afu.h" @@ -31,52 +32,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata) afu->dma_regions =RB_ROOT ; } -/** -* afu_dma_adjust_locked_vm - adjust locked memory -* @dev: port device -* @npages: number of pages -* @incr: increase or decrease locked memory -* -* Increase or decrease the locked memory size with npages input. -* -* Return 0 on success. -* Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK. -*/ -static int afu_dma_adjust_locked_vm (struct device *dev,long npages,bool incr) -{ -unsigned long locked, lock_limit; -int ret =0 ; - -/* the task is exiting. */ -if (!current->mm) -return 0 ; - -down_write (¤t->mm->mmap_sem); - -if (incr) { - locked = current->mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += npages; - }else { -if (WARN_ON_ONCE (npages > current->mm->locked_vm)) - npages = current->mm->locked_vm; - current->mm->locked_vm -= npages; - } - -dev_dbg (dev,"[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n" , current->pid, - incr ?'+' :'-' , npages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT ,rlimit (RLIMIT_MEMLOCK ), - ret ?"- exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -/** * afu_dma_pin_pages - pin pages of given dma memory region * @pdata: feature device platform data @@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,struct device *dev = &pdata->dev->dev;int ret, pinned; - ret =afu_dma_adjust_locked_vm (dev, npages, true); + ret =account_locked_vm (current->mm, npages, true);if (ret)return ret;@@ -121,7 +76,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata, free_pages:kfree (region->pages); unlock_vm: -afu_dma_adjust_locked_vm (dev, npages, false); +account_locked_vm (current->mm, npages, false);return ret; }@@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,put_all_pages (region->pages, npages);kfree (region->pages); -afu_dma_adjust_locked_vm (dev, npages, false); +account_locked_vm (current->mm, npages, false);dev_dbg (dev,"%ld pages unpinned\n" , npages); }drivers/vfio/vfio_iommu_spapr_tce.c +1
-0@@ -22,6 +22,7 @@#include <linux/vmalloc.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> +#include <linux/mm.h> #include <asm/iommu.h> #include <asm/tce.h> @@ -34,49 +35,13 @@ static void tce_iommu_detach_group(void *iommu_data,
drivers/vfio/vfio_iommu_spapr_tce.c line 42
Quoted 10 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2
-3struct iommu_group *iommu_group); -static long try_increment_locked_vm (struct mm_struct *mm,long npages) +static int tce_account_locked_vm (struct mm_struct *mm,unsigned long npages, +bool inc) { -long ret =0 , locked, lock_limit; -if (WARN_ON_ONCE (!mm))return -EPERM ;
If this WARN_ON is the only reason for having tce_account_locked_vm() instead of calling account_locked_vm() directly, you can then ditch the check as I have never ever seen this triggered.
Quoted 90 lines
drivers/vfio/vfio_iommu_spapr_tce.c +6
-40-if (!npages) -return 0 ; - -down_write (&mm->mmap_sem); - locked = mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - mm->locked_vm += npages; - -pr_debug ("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n" , current->pid, - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (&mm->mmap_sem); - -return ret; -} - -static void decrement_locked_vm (struct mm_struct *mm,long npages) -{ -if (!mm || !npages) -return ; - -down_write (&mm->mmap_sem); -if (WARN_ON_ONCE (npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; -pr_debug ("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n" , current->pid, - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK )); -up_write (&mm->mmap_sem); +return account_locked_vm (mm, npages, inc); }/* @@ -336,7 +301,7 @@ static int tce_iommu_enable(struct tce_container *container)return ret; locked = table_group->tce32_size >>PAGE_SHIFT ; - ret =try_increment_locked_vm (container->mm, locked); + ret =tce_account_locked_vm (container->mm, locked, true);if (ret)return ret;@@ -355,7 +320,7 @@ static void tce_iommu_disable(struct tce_container *container) container->enabled = false;BUG_ON (!container->mm); -decrement_locked_vm (container->mm, container->locked_pages); +tce_account_locked_vm (container->mm, container->locked_pages, false); }static void *tce_iommu_open (unsigned long arg)@@ -658,7 +623,8 @@ static long tce_iommu_create_table(struct tce_container *container,if (!table_size)return -EINVAL ; - ret =try_increment_locked_vm (container->mm, table_size >>PAGE_SHIFT ); + ret =tce_account_locked_vm (container->mm, table_size >>PAGE_SHIFT , + true);if (ret)return ret;@@ -677,7 +643,7 @@ static void tce_iommu_free_table(struct tce_container *container,unsigned long pages = tbl->it_allocated_size >>PAGE_SHIFT ;iommu_tce_table_put (tbl); -decrement_locked_vm (container->mm, pages); +tce_account_locked_vm (container->mm, pages, false); } static long tce_iommu_create_window(struct tce_container *container,drivers/vfio/vfio_iommu_type1.c +0
-4@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)return -ESRCH ;/* process exited */ ret =down_write_killable (&mm->mmap_sem); -if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit;
drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277
Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +2
-8- - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit) - ret = -ENOMEM ; - } - } +if (ret) + goto out;
A single "goto" to jump just 3 lines below seems unnecessary.
Quoted 81 lines
drivers/vfio/vfio_iommu_type1.c +4
-5-if (!ret) - mm->locked_vm += npage; - -up_write (&mm->mmap_sem); - } + ret =__account_locked_vm (mm,abs (npage), npage >0 , dma->task, + dma->lock_cap); +up_write (&mm->mmap_sem); +out:if (async)mmput (mm);include/linux/mm.h +19
-0@@ -1521,6 +1521,25 @@ static inline long get_user_pages_longterm(unsigned long start,int get_user_pages_fast (unsigned long start,int nr_pages,int write,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */ mm/util.c +22
-0@@ -6,6 +6,7 @@#include <linux/err.h> #include <linux/sched.h> #include <linux/sched/mm.h> +#include <linux/sched/signal.h> #include <linux/sched/task_stack.h> #include <linux/security.h> #include <linux/swap.h> @@ -346,6 +347,53 @@ int __weak get_user_pages_fast(unsigned long start, }EXPORT_SYMBOL_GPL (get_user_pages_fast); +/** +* __account_locked_vm - account locked pages to an mm's locked_vm +* @mm: mm to account against, may be NULL +* @pages: number of pages to account +* @inc: %true if @pages should be considered positive, %false if not +* @task: task used to check RLIMIT_MEMLOCK +* @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped +* +* Assumes @task and @mm are valid (i.e. at least one reference on each), and +* that mmap_sem is held as writer. +* +* Return: +* * 0 on success +* * 0 if @mm is NULL (can happen for example if the task is exiting) +* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. +*/ +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim) +{ +unsigned long locked_vm, limit; +int ret =0 ;
mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 380
Quoted 10 lines
mm/util.c +10
-0+ + locked_vm = mm->locked_vm; +if (inc) { +if (!bypass_rlim) { + limit =task_rlimit (task,RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; +if (locked_vm + pages > limit) { + ret = -ENOMEM ; + goto out; + } + }
Nit:
if (!ret)
and then you don't need "goto out".
mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 384
Quoted 4 lines
mm/util.c +4
-0+ mm->locked_vm = locked_vm + pages; + }else { +WARN_ON_ONCE (pages > locked_vm); + mm->locked_vm = locked_vm - pages;
Can go negative here. Not a huge deal but inaccurate imo.
Quoted 18 lines
mm/util.c +12
-0+ } + +out: +pr_debug ("%s: [%d] %c%lu %lu/%lu%s\n" , __func__, task->pid, + (inc) ?'+' :'-' , pages <<PAGE_SHIFT , + locked_vm <<PAGE_SHIFT ,task_rlimit (task,RLIMIT_MEMLOCK ), + ret ?" - exceeded" :"" ); + +return ret; +} +EXPORT_SYMBOL_GPL (__account_locked_vm); +unsigned long vm_mmap_pgoff (struct file *file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flag,unsigned long pgoff)base-commit:
37624b58542fb9f2d9a70e6ea006ef8a5f66c30b
Signature · 1 line
Alexey
Quoted 7 lines · Alexey Kardashevskiy
Quoted 3 lines · Daniel Jordan
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.And I rather liked that prints were different and tell precisely which one of three each printk is.
I'm not following. One of three...callsites? But there were five callsites.
Anyway, I added a _RET_IP_ to the debug print so you can differentiate.
Quoted 3 lines
I commented below but in general this seems working.
Tested-by Alexey Kardashevskiy <[email protected]>
Thanks! And for the review as well.
Quoted 8 lines
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +0
-0@@ -34,49 +35,13 @@static void tce_iommu_detach_group (void *iommu_data,struct iommu_group *iommu_group);
Quoted 13 lines
drivers/vfio/vfio_iommu_spapr_tce.c line 42
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2
-3-static long try_increment_locked_vm (struct mm_struct *mm,long npages) +static int tce_account_locked_vm (struct mm_struct *mm,unsigned long npages, +bool inc) { -long ret =0 , locked, lock_limit; -if (WARN_ON_ONCE (!mm))return -EPERM ;If this
WARN_ONis the only reason for havingtce_account_locked_vm()instead of callingaccount_locked_vm()directly, you can then ditch the check as I have never ever seen this triggered.
Great, will do.
Quoted 17 lines
Quoted 17 lines
drivers/vfio/vfio_iommu_type1.c +0
-9@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)return -ESRCH ;/* process exited */ ret =down_write_killable (&mm->mmap_sem); -if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit)
Quoted 8 lines
drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +2
-3- ret = -ENOMEM ; - } - } +if (ret) + goto out;A single "goto" to jump just 3 lines below seems unnecessary.
No strong preference here, I'll take out the goto.
Quoted 18 lines
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +5
-0+int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim) +{ +unsigned long locked_vm, limit; +int ret =0 ;drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +10
-0+ + locked_vm = mm->locked_vm; +if (inc) { +if (!bypass_rlim) { + limit =task_rlimit (task,RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; +if (locked_vm + pages > limit) { + ret = -ENOMEM ; + goto out; + } + }Nit:
Quoted 3 lines
if (!ret)
and then you don't need "goto out".
Ok, sure.
Quoted 7 lines
drivers/vfio/vfio_iommu_type1.c
Quoted 4 lines
drivers/vfio/vfio_iommu_type1.c +4
-0+ mm->locked_vm = locked_vm + pages; + }else { +WARN_ON_ONCE (pages > locked_vm); + mm->locked_vm = locked_vm - pages;Can go negative here. Not a huge deal but inaccurate imo.
I hear you, but setting a negative value to zero, as we had done previously, doesn't make much sense to me.
Quoted 10 lines · Daniel Jordan
Quoted 7 lines · Alexey Kardashevskiy
Quoted 3 lines · Daniel Jordan
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.And I rather liked that prints were different and tell precisely which one of three each printk is.
I'm not following. One of three...callsites? But there were five callsites.
Well, 3 of them are mine, I was referring to them :)
Quoted 1 line
Anyway, I added a _RET_IP_ to the debug print so you can differentiate.
I did not know that existed, cool!
Quoted 91 lines
Quoted 4 lines
I commented below but in general this seems working.
Tested-by Alexey Kardashevskiy <[email protected]>Thanks! And for the review as well.
Quoted 8 lines
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +0
-0@@ -34,49 +35,13 @@static void tce_iommu_detach_group (void *iommu_data,struct iommu_group *iommu_group);Quoted 13 lines
drivers/vfio/vfio_iommu_spapr_tce.c line 42
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2
-3-static long try_increment_locked_vm (struct mm_struct *mm,long npages) +static int tce_account_locked_vm (struct mm_struct *mm,unsigned long npages, +bool inc) { -long ret =0 , locked, lock_limit; -if (WARN_ON_ONCE (!mm))return -EPERM ;If this
WARN_ONis the only reason for havingtce_account_locked_vm()instead of callingaccount_locked_vm()directly, you can then ditch the check as I have never ever seen this triggered.Great, will do.
Quoted 17 lines
Quoted 17 lines
drivers/vfio/vfio_iommu_type1.c +0
-9@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)return -ESRCH ;/* process exited */ ret =down_write_killable (&mm->mmap_sem); -if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit)Quoted 8 lines
drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +2
-3- ret = -ENOMEM ; - } - } +if (ret) + goto out;A single "goto" to jump just 3 lines below seems unnecessary.
No strong preference here, I'll take out the goto.
Quoted 18 lines
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +5
-0+int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim) +{ +unsigned long locked_vm, limit; +int ret =0 ;drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +10
-0+ + locked_vm = mm->locked_vm; +if (inc) { +if (!bypass_rlim) { + limit =task_rlimit (task,RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; +if (locked_vm + pages > limit) { + ret = -ENOMEM ; + goto out; + } + }Nit:
Quoted 3 lines
if (!ret)
and then you don't need "goto out".
Ok, sure.
Quoted 7 lines
drivers/vfio/vfio_iommu_type1.c
Quoted 4 lines
drivers/vfio/vfio_iommu_type1.c +4
-0+ mm->locked_vm = locked_vm + pages; + }else { +WARN_ON_ONCE (pages > locked_vm); + mm->locked_vm = locked_vm - pages;Can go negative here. Not a huge deal but inaccurate imo.
Quoted 2 lines
I hear you, but setting a negative value to zero, as we had done previously, doesn't make much sense to me.
Ok then. I have not seen these WARN_ON for a very long time anyway.
Signature · 1 line
Alexey
PATCH v2 1/1 mm: add account_locked_vm utility function
locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.
Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.
---
Against v5.2-rc1.
v2:
- applied review comments from Alexey
- added _RET_IP_ to debug print to disambiguate callers
arch/powerpc/kvm/book3s_64_vio.c | 44 +++-------------------- arch/powerpc/mm/book3s64/iommu_api.c | 41 +++------------------ drivers/fpga/dfl-afu-dma-region.c | 53 +++------------------------ drivers/vfio/vfio_iommu_spapr_tce.c | 54 +++------------------------- drivers/vfio/vfio_iommu_type1.c | 17 ++------- include/linux/mm.h | 19 ++++++++++ mm/util.c | 46 ++++++++++++++++++++++++ 7 files changed, 84 insertions(+), 190 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
@@ -30,6 +30,7 @@#include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)return tce_pages +ALIGN (stt_bytes,PAGE_SIZE ) /PAGE_SIZE ; } -static long kvmppc_account_memlimit (unsigned long stt_pages,bool inc) -{ -long ret =0 ; - -if (!current || !current->mm) -return ret;/* process exited */ - -down_write (¤t->mm->mmap_sem); - -if (inc) { -unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += stt_pages; - }else { -if (WARN_ON_ONCE (stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n" , current->pid, - inc ?'+' :'-' , - stt_pages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -static void kvm_spapr_tce_iommu_table_free (struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)kvm_put_kvm (stt->kvm); -kvmppc_account_memlimit ( +account_locked_vm (current->mm,kvmppc_stt_pages (kvmppc_tce_pages (stt->size)), false);call_rcu (&stt->rcu, release_spapr_tce_table);
@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,return -EINVAL ; npages =kvmppc_tce_pages (size); - ret =kvmppc_account_memlimit (kvmppc_stt_pages (npages), true); + ret =account_locked_vm (current->mm,kvmppc_stt_pages (npages), true);if (ret)return ret;
@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,kfree (stt); fail_acct: -kvmppc_account_memlimit (kvmppc_stt_pages (npages), false); +account_locked_vm (current->mm,kvmppc_stt_pages (npages), false);return ret; }
arch/powerpc/mm/book3s64/iommu_api.c +4 -37
71 lines not on this page · raw message
drivers/fpga/dfl-afu-dma-region.c +4 -49
83 lines not on this page · raw message
drivers/vfio/vfio_iommu_spapr_tce.c +5 -49
90 lines not on this page · raw message
drivers/vfio/vfio_iommu_type1.c +2 -15
@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) ret =down_write_killable (&mm->mmap_sem);if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit) - ret = -ENOMEM ; - } - } - -if (!ret) - mm->locked_vm += npage; - + ret =__account_locked_vm (mm,abs (npage), npage >0 , dma->task, + dma->lock_cap);up_write (&mm->mmap_sem); }
include/linux/mm.h +19 -0
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */
mm/util.c +46 -0
58 lines not on this page · raw message
base-commit: a188339ca5a396acc588e5851ed7e19f66b0ebd9
Signature · 1 line
2.21.0
Andrew Morton
maintainerQuoted 25 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.Error codes stay the same, so user-visible behavior does too. The one exception is that the
-EPERMcase intce_account_locked_vmis removed because Alexey has never seen it triggered....
include/linux/mm.h +8
-0@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; +
include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584
Quoted 10 lines
include/linux/mm.h +10
-0+if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +}
That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling drivers/vfio/vfio_iommu_type1.c to suit. I wonder why it does down_write_killable and whether it really needs to...
Quoted 34 lines · Andrew Morton
Quoted 32 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.Error codes stay the same, so user-visible behavior does too. The one exception is that the
-EPERMcase intce_account_locked_vmis removed because Alexey has never seen it triggered....
include/linux/mm.h +15
-0@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem);
Quoted 8 lines
include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584
Quoted 3 lines
include/linux/mm.h +3
-0+ +return ret; +}That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling
drivers/vfio/vfio_iommu_type1.cto suit. I wonder why it doesdown_write_killableand whether it really needs to...
Sure, I can uninline it. vfio changelogs don't show a particular reason for _killable[1]. Maybe Alex has something to add. Otherwise I'll respin without it since the simplification seems worth removing _killable.
[1] 0cfef2b7410b ("vfio/type1: Remove locked page accounting workqueue")
Quoted 44 lines · On Tue, 28 May 2019 11:04:24 -0400 Daniel Jordan <[email protected]>…
Quoted 34 lines · Andrew Morton
Quoted 32 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.Error codes stay the same, so user-visible behavior does too. The one exception is that the
-EPERMcase intce_account_locked_vmis removed because Alexey has never seen it triggered....
include/linux/mm.h +15
-0@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem);Quoted 8 lines
include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584
Quoted 3 lines
include/linux/mm.h +3
-0+ +return ret; +}That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling
drivers/vfio/vfio_iommu_type1.cto suit. I wonder why it doesdown_write_killableand whether it really needs to...
Quoted 5 lines
Sure, I can uninline it. vfio changelogs don't show a particular reason for _killable[1]. Maybe Alex has something to add. Otherwise I'll respin without it since the simplification seems worth removing
_killable.[1]
0cfef2b7410b("vfio/type1: Remove locked page accounting workqueue")
A userspace vfio driver maps DMA via an ioctl through this path, so I believe I used killable here just to be friendly that it could be interrupted and we could fall out with an errno if it were stuck here. No harm, no foul, the user's mapping is aborted and unwound. If we're deadlocked or seriously contended on mmap_sem, maybe we're already in trouble, but it seemed like a valid and low hanging use case for killable. Thanks,
Alex
PATCH v3 1/1 mm: add account_locked_vm utility function
locked_vm accounting is done roughly the same way in five places, so unify them in a helper.
Include the helper's caller in the debug print to distinguish between callsites.
Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.
---
v3:
- uninline account_locked_vm (Andrew)
- fix doc comment (Ira)
- retain down_write_killable in vfio type1 (Alex)
- leave Alexey's T-b since the code is the same aside from uninlining
- sanity tested with vfio type1, sanity-built on ppc
arch/powerpc/kvm/book3s_64_vio.c | 44 ++-------------- arch/powerpc/mm/book3s64/iommu_api.c | 41 ++------------- drivers/fpga/dfl-afu-dma-region.c | 53 ++------------------ drivers/vfio/vfio_iommu_spapr_tce.c | 54 ++------------------ drivers/vfio/vfio_iommu_type1.c | 17 +------ include/linux/mm.h | 4 ++ mm/util.c | 75 ++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 190 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
@@ -30,6 +30,7 @@#include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)return tce_pages +ALIGN (stt_bytes,PAGE_SIZE ) /PAGE_SIZE ; } -static long kvmppc_account_memlimit (unsigned long stt_pages,bool inc) -{ -long ret =0 ; - -if (!current || !current->mm) -return ret;/* process exited */ - -down_write (¤t->mm->mmap_sem); - -if (inc) { -unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += stt_pages; - }else { -if (WARN_ON_ONCE (stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n" , current->pid, - inc ?'+' :'-' , - stt_pages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -static void kvm_spapr_tce_iommu_table_free (struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)kvm_put_kvm (stt->kvm); -kvmppc_account_memlimit ( +account_locked_vm (current->mm,kvmppc_stt_pages (kvmppc_tce_pages (stt->size)), false);call_rcu (&stt->rcu, release_spapr_tce_table);
@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,return -EINVAL ; npages =kvmppc_tce_pages (size); - ret =kvmppc_account_memlimit (kvmppc_stt_pages (npages), true); + ret =account_locked_vm (current->mm,kvmppc_stt_pages (npages), true);if (ret)return ret;
@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,kfree (stt); fail_acct: -kvmppc_account_memlimit (kvmppc_stt_pages (npages), false); +account_locked_vm (current->mm,kvmppc_stt_pages (npages), false);return ret; }
arch/powerpc/mm/book3s64/iommu_api.c +4 -37
71 lines not on this page · raw message
drivers/fpga/dfl-afu-dma-region.c +4 -49
83 lines not on this page · raw message
drivers/vfio/vfio_iommu_spapr_tce.c +5 -49
90 lines not on this page · raw message
drivers/vfio/vfio_iommu_type1.c +2 -15
@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) ret =down_write_killable (&mm->mmap_sem);if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit) - ret = -ENOMEM ; - } - } - -if (!ret) - mm->locked_vm += npage; - + ret =__account_locked_vm (mm,abs (npage), npage >0 , dma->task, + dma->lock_cap);up_write (&mm->mmap_sem); }
include/linux/mm.h +4 -0
@@ -1564,6 +1564,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */
mm/util.c +75 -0
87 lines not on this page · raw message
base-commit: cd6c84d8f0cdc911df435bb075ba22ce3c605b07
Signature · 1 line
2.21.0
Quoted 41 lines · On Wed, 29 May 2019 16:50:19 -0400 Daniel Jordan <[email protected]>…
locked_vmaccounting is done roughly the same way in five places, so unify them in a helper.Include the helper's caller in the debug print to distinguish between callsites.
Error codes stay the same, so user-visible behavior does too. The one exception is that the
-EPERMcase intce_account_locked_vmis removed because Alexey has never seen it triggered.Signed-off-by Daniel Jordan <[email protected]>Tested-by Alexey Kardashevskiy <[email protected]>Cc Alan Tull <[email protected]>Cc Alex Williamson <[email protected]>Cc Andrew Morton <[email protected]>Cc Benjamin Herrenschmidt <[email protected]>Cc Christoph Lameter <[email protected]>Cc Christophe Leroy <[email protected]>Cc Davidlohr Bueso <[email protected]>Cc Ira Weiny <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc Mark Rutland <[email protected]>Cc Michael Ellerman <[email protected]>Cc Moritz Fischer <[email protected]>Cc Paul Mackerras <[email protected]>Cc Steve Sistare <[email protected]>Cc Wu Hao <[email protected]>---
v3:
- uninlineaccount_locked_vm(Andrew)
- fix doc comment (Ira)
- retaindown_write_killablein vfio type1 (Alex)
- leave Alexey's T-b since the code is the same aside from uninlining
- sanity tested with vfio type1, sanity-built on ppc
Quoted 8 lines
arch/powerpc/kvm/book3s_64_vio.c | 44 ++-------------- arch/powerpc/mm/book3s64/iommu_api.c | 41 ++------------- drivers/fpga/dfl-afu-dma-region.c | 53 ++------------------ drivers/vfio/vfio_iommu_spapr_tce.c | 54 ++------------------ drivers/vfio/vfio_iommu_type1.c | 17 +------ include/linux/mm.h | 4 ++ mm/util.c | 75 ++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 190 deletions(-)
I tend to prefer adding a negative rather than converting to absolute and passing a bool for inc/dec, but it all seems equivalent, so for vfio parts
Quoted 494 lines
arch/powerpc/kvm/book3s_64_vio.c +4
-40@@ -30,6 +30,7 @@#include <linux/anon_inodes.h> #include <linux/iommu.h> #include <linux/file.h> +#include <linux/mm.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> @@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)return tce_pages +ALIGN (stt_bytes,PAGE_SIZE ) /PAGE_SIZE ; } -static long kvmppc_account_memlimit (unsigned long stt_pages,bool inc) -{ -long ret =0 ; - -if (!current || !current->mm) -return ret;/* process exited */ - -down_write (¤t->mm->mmap_sem); - -if (inc) { -unsigned long locked, lock_limit; - - locked = current->mm->locked_vm + stt_pages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += stt_pages; - }else { -if (WARN_ON_ONCE (stt_pages > current->mm->locked_vm)) - stt_pages = current->mm->locked_vm; - - current->mm->locked_vm -= stt_pages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n" , current->pid, - inc ?'+' :'-' , - stt_pages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -static void kvm_spapr_tce_iommu_table_free (struct rcu_head *head) { struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)kvm_put_kvm (stt->kvm); -kvmppc_account_memlimit ( +account_locked_vm (current->mm,kvmppc_stt_pages (kvmppc_tce_pages (stt->size)), false);call_rcu (&stt->rcu, release_spapr_tce_table);@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,return -EINVAL ; npages =kvmppc_tce_pages (size); - ret =kvmppc_account_memlimit (kvmppc_stt_pages (npages), true); + ret =account_locked_vm (current->mm,kvmppc_stt_pages (npages), true);if (ret)return ret;@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,kfree (stt); fail_acct: -kvmppc_account_memlimit (kvmppc_stt_pages (npages), false); +account_locked_vm (current->mm,kvmppc_stt_pages (npages), false);return ret; }arch/powerpc/mm/book3s64/iommu_api.c +4
-37@@ -19,6 +19,7 @@#include <linux/hugetlb.h> #include <linux/swap.h> #include <linux/sizes.h> +#include <linux/mm.h> #include <asm/mmu_context.h> #include <asm/pte-walk.h> #include <linux/mm_inline.h> @@ -51,40 +52,6 @@ struct mm_iommu_table_group_mem_t {u64 dev_hpa;/* Device memory base address */ }; -static long mm_iommu_adjust_locked_vm (struct mm_struct *mm, -unsigned long npages,bool incr) -{ -long ret =0 , locked, lock_limit; - -if (!npages) -return 0 ; - -down_write (&mm->mmap_sem); - -if (incr) { - locked = mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - mm->locked_vm += npages; - }else { -if (WARN_ON_ONCE (npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; - } - -pr_debug ("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n" , - current ? current->pid :0 , - incr ?'+' :'-' , - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK )); -up_write (&mm->mmap_sem); - -return ret; -} -bool mm_iommu_preregistered (struct mm_struct *mm) {return !list_empty (&mm->context.iommu_group_mem_list);@@ -101,7 +68,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,unsigned long entry, chunk;if (dev_hpa =MM_IOMMU_TABLE_INVALID_HPA ) { - ret =mm_iommu_adjust_locked_vm (mm, entries, true); + ret =account_locked_vm (mm, entries, true);if (ret)return ret;@@ -216,7 +183,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,kfree (mem); unlock_exit: -mm_iommu_adjust_locked_vm (mm, locked_entries, false); +account_locked_vm (mm, locked_entries, false);return ret; }@@ -316,7 +283,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) unlock_exit:mutex_unlock (&mem_list_mutex); -mm_iommu_adjust_locked_vm (mm, unlock_entries, false); +account_locked_vm (mm, unlock_entries, false);return ret; }drivers/fpga/dfl-afu-dma-region.c +4
-49@@ -12,6 +12,7 @@#include <linux/dma-mapping.h> #include <linux/sched/signal.h> #include <linux/uaccess.h> +#include <linux/mm.h> #include "dfl-afu.h" @@ -31,52 +32,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata) afu->dma_regions =RB_ROOT ; } -/** -* afu_dma_adjust_locked_vm - adjust locked memory -* @dev: port device -* @npages: number of pages -* @incr: increase or decrease locked memory -* -* Increase or decrease the locked memory size with npages input. -* -* Return 0 on success. -* Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK. -*/ -static int afu_dma_adjust_locked_vm (struct device *dev,long npages,bool incr) -{ -unsigned long locked, lock_limit; -int ret =0 ; - -/* the task is exiting. */ -if (!current->mm) -return 0 ; - -down_write (¤t->mm->mmap_sem); - -if (incr) { - locked = current->mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - current->mm->locked_vm += npages; - }else { -if (WARN_ON_ONCE (npages > current->mm->locked_vm)) - npages = current->mm->locked_vm; - current->mm->locked_vm -= npages; - } - -dev_dbg (dev,"[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n" , current->pid, - incr ?'+' :'-' , npages <<PAGE_SHIFT , - current->mm->locked_vm <<PAGE_SHIFT ,rlimit (RLIMIT_MEMLOCK ), - ret ?"- exceeded" :"" ); - -up_write (¤t->mm->mmap_sem); - -return ret; -} -/** * afu_dma_pin_pages - pin pages of given dma memory region * @pdata: feature device platform data @@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,struct device *dev = &pdata->dev->dev;int ret, pinned; - ret =afu_dma_adjust_locked_vm (dev, npages, true); + ret =account_locked_vm (current->mm, npages, true);if (ret)return ret;@@ -121,7 +76,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata, free_pages:kfree (region->pages); unlock_vm: -afu_dma_adjust_locked_vm (dev, npages, false); +account_locked_vm (current->mm, npages, false);return ret; }@@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,put_all_pages (region->pages, npages);kfree (region->pages); -afu_dma_adjust_locked_vm (dev, npages, false); +account_locked_vm (current->mm, npages, false);dev_dbg (dev,"%ld pages unpinned\n" , npages); }drivers/vfio/vfio_iommu_spapr_tce.c +5
-49@@ -22,6 +22,7 @@#include <linux/vmalloc.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> +#include <linux/mm.h> #include <asm/iommu.h> #include <asm/tce.h> @@ -34,51 +35,6 @@static void tce_iommu_detach_group (void *iommu_data,struct iommu_group *iommu_group); -static long try_increment_locked_vm (struct mm_struct *mm,long npages) -{ -long ret =0 , locked, lock_limit; - -if (WARN_ON_ONCE (!mm)) -return -EPERM ; - -if (!npages) -return 0 ; - -down_write (&mm->mmap_sem); - locked = mm->locked_vm + npages; - lock_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; -if (locked > lock_limit && !capable (CAP_IPC_LOCK )) - ret = -ENOMEM ; -else - mm->locked_vm += npages; - -pr_debug ("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n" , current->pid, - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK ), - ret ?" - exceeded" :"" ); - -up_write (&mm->mmap_sem); - -return ret; -} - -static void decrement_locked_vm (struct mm_struct *mm,long npages) -{ -if (!mm || !npages) -return ; - -down_write (&mm->mmap_sem); -if (WARN_ON_ONCE (npages > mm->locked_vm)) - npages = mm->locked_vm; - mm->locked_vm -= npages; -pr_debug ("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n" , current->pid, - npages <<PAGE_SHIFT , - mm->locked_vm <<PAGE_SHIFT , -rlimit (RLIMIT_MEMLOCK )); -up_write (&mm->mmap_sem); -} -/* * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation * @@ -336,7 +292,7 @@ static int tce_iommu_enable(struct tce_container *container)return ret; locked = table_group->tce32_size >>PAGE_SHIFT ; - ret =try_increment_locked_vm (container->mm, locked); + ret =account_locked_vm (container->mm, locked, true);if (ret)return ret;@@ -355,7 +311,7 @@ static void tce_iommu_disable(struct tce_container *container) container->enabled = false;BUG_ON (!container->mm); -decrement_locked_vm (container->mm, container->locked_pages); +account_locked_vm (container->mm, container->locked_pages, false); }static void *tce_iommu_open (unsigned long arg)@@ -659,7 +615,7 @@ static long tce_iommu_create_table(struct tce_container *container,if (!table_size)return -EINVAL ; - ret =try_increment_locked_vm (container->mm, table_size >>PAGE_SHIFT ); + ret =account_locked_vm (container->mm, table_size >>PAGE_SHIFT , true);if (ret)return ret;@@ -678,7 +634,7 @@ static void tce_iommu_free_table(struct tce_container *container,unsigned long pages = tbl->it_allocated_size >>PAGE_SHIFT ;iommu_tce_table_put (tbl); -decrement_locked_vm (container->mm, pages); +account_locked_vm (container->mm, pages, false); } static long tce_iommu_create_window(struct tce_container *container,drivers/vfio/vfio_iommu_type1.c +2
-15@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) ret =down_write_killable (&mm->mmap_sem);if (!ret) { -if (npage >0 ) { -if (!dma->lock_cap) { -unsigned long limit; - - limit =task_rlimit (dma->task, -RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; - -if (mm->locked_vm + npage > limit) - ret = -ENOMEM ; - } - } - -if (!ret) - mm->locked_vm += npage; - + ret =__account_locked_vm (mm,abs (npage), npage >0 , dma->task, + dma->lock_cap);up_write (&mm->mmap_sem); }include/linux/mm.h +4
-0@@ -1564,6 +1564,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */ mm/util.c +75
-0@@ -7,6 +7,7 @@#include <linux/err.h> #include <linux/sched.h> #include <linux/sched/mm.h> +#include <linux/sched/signal.h> #include <linux/sched/task_stack.h> #include <linux/security.h> #include <linux/swap.h> @@ -347,6 +348,80 @@ int __weak get_user_pages_fast(unsigned long start, }EXPORT_SYMBOL_GPL (get_user_pages_fast); +/** +* __account_locked_vm - account locked pages to an mm's locked_vm +* @mm: mm to account against +* @pages: number of pages to account +* @inc: %true if @pages should be considered positive, %false if not +* @task: task used to check RLIMIT_MEMLOCK +* @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped +* +* Assumes @task and @mm are valid (i.e. at least one reference on each), and +* that mmap_sem is held as writer. +* +* Return: +* * 0 on success +* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. +*/ +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim) +{ +unsigned long locked_vm, limit; +int ret =0 ; + +lockdep_assert_held_exclusive (&mm->mmap_sem); + + locked_vm = mm->locked_vm; +if (inc) { +if (!bypass_rlim) { + limit =task_rlimit (task,RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; +if (locked_vm + pages > limit) + ret = -ENOMEM ; + } +if (!ret) + mm->locked_vm = locked_vm + pages; + }else { +WARN_ON_ONCE (pages > locked_vm); + mm->locked_vm = locked_vm - pages; + } + +pr_debug ("%s: [%d] caller %ps %c%lu %lu/%lu%s\n" , __func__, task->pid, + (void *)_RET_IP_, (inc) ?'+' :'-' , pages <<PAGE_SHIFT , + locked_vm <<PAGE_SHIFT ,task_rlimit (task,RLIMIT_MEMLOCK ), + ret ?" - exceeded" :"" ); + +return ret; +} +EXPORT_SYMBOL_GPL (__account_locked_vm); + +/** +* account_locked_vm - account locked pages to an mm's locked_vm +* @mm: mm to account against, may be NULL +* @pages: number of pages to account +* @inc: %true if @pages should be considered positive, %false if not +* +* Assumes a non-NULL @mm is valid (i.e. at least one reference on it). +* +* Return: +* * 0 on success, or if mm is NULL +* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. +*/ +int account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +EXPORT_SYMBOL_GPL (account_locked_vm); +unsigned long vm_mmap_pgoff (struct file *file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flag,unsigned long pgoff)base-commit:
cd6c84d8f0cdc911df435bb075ba22ce3c605b07
On Fri, May 24, 2019 at 01:50:45PM -0400, Daniel Jordan wrote:
[snip]
Quoted 43 lines
include/linux/mm.h +19
-0@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,int get_user_pages_fast (unsigned long start,int nr_pages,unsigned int gup_flags,struct page **pages); +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim); + +static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages ==0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +/* Container for pinned pfns / pages */ struct frame_vector {unsigned int nr_allocated;/* Number of frames we have space for */ mm/util.c +1
-0@@ -6,6 +6,7 @@#include <linux/err.h> #include <linux/sched.h> #include <linux/sched/mm.h> +#include <linux/sched/signal.h> #include <linux/sched/task_stack.h> #include <linux/security.h> #include <linux/swap.h>
mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 352
Quoted 6 lines
mm/util.c +3
-0@@ -346,6 +347,51 @@ int __weak get_user_pages_fast(unsigned long start, }EXPORT_SYMBOL_GPL (get_user_pages_fast); +/** +* __account_locked_vm - account locked pages to an mm's locked_vm +* @mm: mm to account against, may be NULL
This kernel doc is wrong. You dereference mm straight away...
Quoted 10 lines
mm/util.c +10
-0+* @pages: number of pages to account +* @inc: %true if @pages should be considered positive, %false if not +* @task: task used to check RLIMIT_MEMLOCK +* @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped +* +* Assumes @task and @mm are valid (i.e. at least one reference on each), and +* that mmap_sem is held as writer. +* +* Return: +* * 0 on success
mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 372
Quoted 10 lines
mm/util.c +10
-0+* * 0 if @mm is NULL (can happen for example if the task is exiting) +* * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. +*/ +int __account_locked_vm (struct mm_struct *mm,unsigned long pages,bool inc, +struct task_struct *task,bool bypass_rlim) +{ +unsigned long locked_vm, limit; +int ret =0 ; + + locked_vm = mm->locked_vm;
here...
Perhaps the comment was meant to document account_locked_vm()? Or should the parameter checks be moved here?
Ira
Quoted 32 lines
mm/util.c +22
-0+if (inc) { +if (!bypass_rlim) { + limit =task_rlimit (task,RLIMIT_MEMLOCK ) >>PAGE_SHIFT ; +if (locked_vm + pages > limit) + ret = -ENOMEM ; + } +if (!ret) + mm->locked_vm = locked_vm + pages; + }else { +WARN_ON_ONCE (pages > locked_vm); + mm->locked_vm = locked_vm - pages; + } + +pr_debug ("%s: [%d] caller %ps %c%lu %lu/%lu%s\n" , __func__, task->pid, + (void *)_RET_IP_, (inc) ?'+' :'-' , pages <<PAGE_SHIFT , + locked_vm <<PAGE_SHIFT ,task_rlimit (task,RLIMIT_MEMLOCK ), + ret ?" - exceeded" :"" ); + +return ret; +} +EXPORT_SYMBOL_GPL (__account_locked_vm); + unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,mm/util.c +0
-0unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff)base-commit:
a188339ca5a396acc588e5851ed7e19f66b0ebd9Signature · 1 line
2.21.0
Quoted 17 lines · Ira Weiny
Quoted 16 lines · Daniel Jordan
include/linux/mm.h +16
-0+static inline int account_locked_vm (struct mm_struct *mm,unsigned long pages, +bool inc) +{ +int ret; + +if (pages =0 || !mm) +return 0 ; + +down_write (&mm->mmap_sem); + ret =__account_locked_vm (mm, pages, inc, current, +capable (CAP_IPC_LOCK )); +up_write (&mm->mmap_sem); + +return ret; +} +
...snip...
Quoted 5 lines
Quoted 3 lines
+/**
+ *__account_locked_vm- account locked pages to an mm'slocked_vm
+ * @mm: mm to account against, may be NULLThis kernel doc is wrong. You dereference mm straight away...
...snip...
Quoted 6 lines
mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 372
Quoted 2 lines
mm/util.c +2
-0+ + locked_vm = mm->locked_vm;here...
Perhaps the comment was meant to document
account_locked_vm()?
Yes, the comment got out of sync when I moved the !mm check outside __account_locked_vm. Thanks for catching, will fix.