Skip to contentSkip to conversation

mm: add account_locked_vm utility function

PATCH Superseded Daniel Jordan 6 participants 14 messages May 3, 2019 – Jun 3, 2019

Subsystems (9)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mm: add account_locked_vm utility function

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly. Error codes stay the same, so user-visible behavior does too.

Cc Alan Tull <[email protected]>
Cc Alexey Kardashevskiy <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

---

Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.

Andrew, this one patch replaces these six from [1]:

mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch

That series converts locked_vm to an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers use pinned_vm instead.

Christophe's suggestion of cmpxchg[2] does prevent the races he mentioned for pinned_vm, but others would still remain. In perf_mmap and the hfi1 driver, pinned_vm is checked against the rlimit racily and then later increased when the pinned_vm originally read may have gone stale. Any fixes for that, that I could think of, seem about as good as what's there now, so I left it. I have a patch that uses cmpxchg with pinned_vm if others feel strongly that the aforementioned races should be fixed.

Daniel

[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/

 arch/powerpc/kvm/book3s_64_vio.c    | 44 +++---------------------
 arch/powerpc/mm/mmu_context_iommu.c | 41 +++-------------------
 drivers/fpga/dfl-afu-dma-region.c   | 53 +++--------------------------
 drivers/vfio/vfio_iommu_spapr_tce.c | 52 +++++-----------------------
 drivers/vfio/vfio_iommu_type1.c     | 23 ++++---------
 include/linux/mm.h                  | 19 +++++++++++
 mm/util.c                           | 48 ++++++++++++++++++++++++++
 7 files changed, 94 insertions(+), 186 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
index f02b04973710..f7d37fa6003a 100644
@@ -30,6 +30,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/iommu.h>
 #include <linux/file.h>
+#include <linux/mm.h>
 
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
 	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
 }
 
-static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
-{
-	long ret = 0;
-
-	if (!current || !current->mm)
-		return ret; /* process exited */
-
-	down_write(&current->mm->mmap_sem);
-
-	if (inc) {
-		unsigned long locked, lock_limit;
-
-		locked = current->mm->locked_vm + stt_pages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += stt_pages;
-	} else {
-		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
-			stt_pages = current->mm->locked_vm;
-
-		current->mm->locked_vm -= stt_pages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
-			inc ? '+' : '-',
-			stt_pages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
 {
 	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -277,7 +241,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
 
 	kvm_put_kvm(stt->kvm);
 
-	kvmppc_account_memlimit(
+	account_locked_vm(current->mm,
 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
 	call_rcu(&stt->rcu, release_spapr_tce_table);
 
@@ -303,7 +267,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 		return -EINVAL;
 
 	npages = kvmppc_tce_pages(size);
-	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
+	ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
 	if (ret)
 		return ret;
 
@@ -359,7 +323,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 
 	kfree(stt);
  fail_acct:
-	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
+	account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
 	return ret;
 }
 
arch/powerpc/mm/mmu_context_iommu.c +4 -37

71 lines not on this page · raw message

drivers/fpga/dfl-afu-dma-region.c +4 -49

83 lines not on this page · raw message

drivers/vfio/vfio_iommu_spapr_tce.c +9 -43

92 lines not on this page · raw message

drivers/vfio/vfio_iommu_type1.c +6 -17
index d0f731c9920a..15ac76171ccd 100644
@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 		return -ESRCH; /* process exited */
 
 	ret = down_write_killable(&mm->mmap_sem);
-	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
-					ret = -ENOMEM;
-			}
-		}
+	if (ret)
+		goto out;
 
-		if (!ret)
-			mm->locked_vm += npage;
-
-		up_write(&mm->mmap_sem);
-	}
+	ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
+				  dma->lock_cap);
+	up_write(&mm->mmap_sem);
 
+out:
 	if (async)
 		mmput(mm);
 
include/linux/mm.h +19 -0
index 6b10c21630f5..7134e55ca23f 100644
@@ -1521,6 +1521,25 @@ static inline long get_user_pages_longterm(unsigned long start,
 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 			struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +48 -0

60 lines not on this page · raw message

base-commit: 37624b58542fb9f2d9a70e6ea006ef8a5f66c30b

Signature · 1 line
2.21.0
#1 lore raw
Quoted 39 lines · Daniel Jordan

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly. Error codes stay the same, so user-visible behavior does too.

Cc Alan Tull <[email protected]>
Cc Alexey Kardashevskiy <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.

Andrew, this one patch replaces these six from [1]:

mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch

Quoted 4 lines

That series converts locked_vm to an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers use pinned_vm instead.

Did we ever decide what to do here? Should all these drivers be switched to pinned_vm anyhow?

Jason

#2 lore raw
Quoted 10 lines · Jason Gunthorpe
Quoted 9 lines · Daniel Jordan

Andrew, this one patch replaces these six from [1]:

mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch

Quoted 7 lines
Quoted 4 lines

That series converts locked_vm to an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers use pinned_vm instead.

Did we ever decide what to do here? Should all these drivers be switched to pinned_vm anyhow?

Well, there were the concerns about switching in [1]. Alex, is there an example of an application or library that would break or be exploitable? If there were particular worries (qemu for vfio type1, for example), perhaps some coordinated changes across the kernel and userspace would be possible, especially given the amount of effort it's likely going to take to get the locked_vm/pinned_vm accounting sorted out.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Tested-by on v1 1/1
#3 lore raw
In reply to Daniel Jordan
Quoted 3 lines · Daniel Jordan

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.

And I rather liked that prints were different and tell precisely which one of three each printk is.

I commented below but in general this seems working.

Quoted 335 lines

Error codes stay the same, so user-visible behavior does too.

Cc Alan Tull <[email protected]>
Cc Alexey Kardashevskiy <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

---

Based on v5.1-rc7. Tested with the vfio type1 driver. Any feedback welcome.

Andrew, this one patch replaces these six from [1]:

mm-change-locked_vms-type-from-unsigned-long-to-atomic64_t.patch vfio-type1-drop-mmap_sem-now-that-locked_vm-is-atomic.patch vfio-spapr_tce-drop-mmap_sem-now-that-locked_vm-is-atomic.patch fpga-dlf-afu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch kvm-book3s-drop-mmap_sem-now-that-locked_vm-is-atomic.patch powerpc-mmu-drop-mmap_sem-now-that-locked_vm-is-atomic.patch

That series converts locked_vm to an atomic, but on closer inspection causes at least one accounting race in mremap, and fixing it just for this type conversion came with too much ugly in the core mm to justify, especially when the right long-term fix is making these drivers use pinned_vm instead.

Christophe's suggestion of cmpxchg[2] does prevent the races he mentioned for pinned_vm, but others would still remain. In perf_mmap and the hfi1 driver, pinned_vm is checked against the rlimit racily and then later increased when the pinned_vm originally read may have gone stale. Any fixes for that, that I could think of, seem about as good as what's there now, so I left it. I have a patch that uses cmpxchg with pinned_vm if others feel strongly that the aforementioned races should be fixed.

Daniel

[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/

 arch/powerpc/kvm/book3s_64_vio.c    | 44 +++---------------------
 arch/powerpc/mm/mmu_context_iommu.c | 41 +++-------------------
 drivers/fpga/dfl-afu-dma-region.c   | 53 +++--------------------------
 drivers/vfio/vfio_iommu_spapr_tce.c | 52 +++++-----------------------
 drivers/vfio/vfio_iommu_type1.c     | 23 ++++---------
 include/linux/mm.h                  | 19 +++++++++++
 mm/util.c                           | 48 ++++++++++++++++++++++++++
 7 files changed, 94 insertions(+), 186 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
index f02b04973710..f7d37fa6003a 100644
@@ -30,6 +30,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/iommu.h>
 #include <linux/file.h>
+#include <linux/mm.h>
 
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
 	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
 }
 
-static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
-{
-	long ret = 0;
-
-	if (!current || !current->mm)
-		return ret; /* process exited */
-
-	down_write(&current->mm->mmap_sem);
-
-	if (inc) {
-		unsigned long locked, lock_limit;
-
-		locked = current->mm->locked_vm + stt_pages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += stt_pages;
-	} else {
-		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
-			stt_pages = current->mm->locked_vm;
-
-		current->mm->locked_vm -= stt_pages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
-			inc ? '+' : '-',
-			stt_pages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
 {
 	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -277,7 +241,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
 
 	kvm_put_kvm(stt->kvm);
 
-	kvmppc_account_memlimit(
+	account_locked_vm(current->mm,
 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
 	call_rcu(&stt->rcu, release_spapr_tce_table);
 
@@ -303,7 +267,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 		return -EINVAL;
 
 	npages = kvmppc_tce_pages(size);
-	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
+	ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
 	if (ret)
 		return ret;
 
@@ -359,7 +323,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 
 	kfree(stt);
  fail_acct:
-	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
+	account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
 	return ret;
 }
 
arch/powerpc/mm/mmu_context_iommu.c +4 -37
index 8330f135294f..9e7001a70570 100644
@@ -19,6 +19,7 @@
 #include <linux/hugetlb.h>
 #include <linux/swap.h>
 #include <linux/sizes.h>
+#include <linux/mm.h>
 #include <asm/mmu_context.h>
 #include <asm/pte-walk.h>
 #include <linux/mm_inline.h>
@@ -51,40 +52,6 @@ struct mm_iommu_table_group_mem_t {
 	u64 dev_hpa;		/* Device memory base address */
 };
 
-static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
-		unsigned long npages, bool incr)
-{
-	long ret = 0, locked, lock_limit;
-
-	if (!npages)
-		return 0;
-
-	down_write(&mm->mmap_sem);
-
-	if (incr) {
-		locked = mm->locked_vm + npages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			mm->locked_vm += npages;
-	} else {
-		if (WARN_ON_ONCE(npages > mm->locked_vm))
-			npages = mm->locked_vm;
-		mm->locked_vm -= npages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
-			current ? current->pid : 0,
-			incr ? '+' : '-',
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK));
-	up_write(&mm->mmap_sem);
-
-	return ret;
-}
-
 bool mm_iommu_preregistered(struct mm_struct *mm)
 {
 	return !list_empty(&mm->context.iommu_group_mem_list);
@@ -101,7 +68,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
 	unsigned long entry, chunk;
 
 	if (dev_hpa = MM_IOMMU_TABLE_INVALID_HPA) {
-		ret = mm_iommu_adjust_locked_vm(mm, entries, true);
+		ret = account_locked_vm(mm, entries, true);
 		if (ret)
 			return ret;
 
@@ -215,7 +182,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
 	kfree(mem);
 
 unlock_exit:
-	mm_iommu_adjust_locked_vm(mm, locked_entries, false);
+	account_locked_vm(mm, locked_entries, false);
 
 	return ret;
 }
@@ -315,7 +282,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
 unlock_exit:
 	mutex_unlock(&mem_list_mutex);
 
-	mm_iommu_adjust_locked_vm(mm, unlock_entries, false);
+	account_locked_vm(mm, unlock_entries, false);
 
 	return ret;
 }
drivers/fpga/dfl-afu-dma-region.c +4 -49
index e18a786fc943..059438e17193 100644
@@ -12,6 +12,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/sched/signal.h>
 #include <linux/uaccess.h>
+#include <linux/mm.h>
 
 #include "dfl-afu.h"
 
@@ -31,52 +32,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
 	afu->dma_regions = RB_ROOT;
 }
 
-/**
- * afu_dma_adjust_locked_vm - adjust locked memory
- * @dev: port device
- * @npages: number of pages
- * @incr: increase or decrease locked memory
- *
- * Increase or decrease the locked memory size with npages input.
- *
- * Return 0 on success.
- * Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK.
- */
-static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr)
-{
-	unsigned long locked, lock_limit;
-	int ret = 0;
-
-	/* the task is exiting. */
-	if (!current->mm)
-		return 0;
-
-	down_write(&current->mm->mmap_sem);
-
-	if (incr) {
-		locked = current->mm->locked_vm + npages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += npages;
-	} else {
-		if (WARN_ON_ONCE(npages > current->mm->locked_vm))
-			npages = current->mm->locked_vm;
-		current->mm->locked_vm -= npages;
-	}
-
-	dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid,
-		incr ? '+' : '-', npages << PAGE_SHIFT,
-		current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK),
-		ret ? "- exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 /**
  * afu_dma_pin_pages - pin pages of given dma memory region
  * @pdata: feature device platform data
@@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
 	struct device *dev = &pdata->dev->dev;
 	int ret, pinned;
 
-	ret = afu_dma_adjust_locked_vm(dev, npages, true);
+	ret = account_locked_vm(current->mm, npages, true);
 	if (ret)
 		return ret;
 
@@ -121,7 +76,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
 free_pages:
 	kfree(region->pages);
 unlock_vm:
-	afu_dma_adjust_locked_vm(dev, npages, false);
+	account_locked_vm(current->mm, npages, false);
 	return ret;
 }
 
@@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
 
 	put_all_pages(region->pages, npages);
 	kfree(region->pages);
-	afu_dma_adjust_locked_vm(dev, npages, false);
+	account_locked_vm(current->mm, npages, false);
 
 	dev_dbg(dev, "%ld pages unpinned\n", npages);
 }
drivers/vfio/vfio_iommu_spapr_tce.c +1 -0
index 6b64e45a5269..d39a1b830d82 100644
@@ -22,6 +22,7 @@
 #include <linux/vmalloc.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
+#include <linux/mm.h>
 
 #include <asm/iommu.h>
 #include <asm/tce.h>
@@ -34,49 +35,13 @@
 static void tce_iommu_detach_group(void *iommu_data,

drivers/vfio/vfio_iommu_spapr_tce.c line 42

Quoted 10 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2 -3
 		struct iommu_group *iommu_group);
 
-static long try_increment_locked_vm(struct mm_struct *mm, long npages)
+static int tce_account_locked_vm(struct mm_struct *mm, unsigned long npages,
+				 bool inc)
 {
-	long ret = 0, locked, lock_limit;
-
 	if (WARN_ON_ONCE(!mm))
 		return -EPERM;

If this WARN_ON is the only reason for having tce_account_locked_vm() instead of calling account_locked_vm() directly, you can then ditch the check as I have never ever seen this triggered.

Quoted 90 lines
drivers/vfio/vfio_iommu_spapr_tce.c +6 -40
-	if (!npages)
-		return 0;
-
-	down_write(&mm->mmap_sem);
-	locked = mm->locked_vm + npages;
-	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-		ret = -ENOMEM;
-	else
-		mm->locked_vm += npages;
-
-	pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid,
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&mm->mmap_sem);
-
-	return ret;
-}
-
-static void decrement_locked_vm(struct mm_struct *mm, long npages)
-{
-	if (!mm || !npages)
-		return;
-
-	down_write(&mm->mmap_sem);
-	if (WARN_ON_ONCE(npages > mm->locked_vm))
-		npages = mm->locked_vm;
-	mm->locked_vm -= npages;
-	pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid,
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK));
-	up_write(&mm->mmap_sem);
+	return account_locked_vm(mm, npages, inc);
 }
 
 /*
@@ -336,7 +301,7 @@ static int tce_iommu_enable(struct tce_container *container)
 		return ret;
 
 	locked = table_group->tce32_size >> PAGE_SHIFT;
-	ret = try_increment_locked_vm(container->mm, locked);
+	ret = tce_account_locked_vm(container->mm, locked, true);
 	if (ret)
 		return ret;
 
@@ -355,7 +320,7 @@ static void tce_iommu_disable(struct tce_container *container)
 	container->enabled = false;
 
 	BUG_ON(!container->mm);
-	decrement_locked_vm(container->mm, container->locked_pages);
+	tce_account_locked_vm(container->mm, container->locked_pages, false);
 }
 
 static void *tce_iommu_open(unsigned long arg)
@@ -658,7 +623,8 @@ static long tce_iommu_create_table(struct tce_container *container,
 	if (!table_size)
 		return -EINVAL;
 
-	ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT);
+	ret = tce_account_locked_vm(container->mm, table_size >> PAGE_SHIFT,
+				    true);
 	if (ret)
 		return ret;
 
@@ -677,7 +643,7 @@ static void tce_iommu_free_table(struct tce_container *container,
 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
 
 	iommu_tce_table_put(tbl);
-	decrement_locked_vm(container->mm, pages);
+	tce_account_locked_vm(container->mm, pages, false);
 }
 
 static long tce_iommu_create_window(struct tce_container *container,
drivers/vfio/vfio_iommu_type1.c +0 -4
index d0f731c9920a..15ac76171ccd 100644
@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 		return -ESRCH; /* process exited */
 
 	ret = down_write_killable(&mm->mmap_sem);
-	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;

drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277

Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +2 -8
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
-					ret = -ENOMEM;
-			}
-		}
+	if (ret)
+		goto out;

A single "goto" to jump just 3 lines below seems unnecessary.

Quoted 81 lines
drivers/vfio/vfio_iommu_type1.c +4 -5
-		if (!ret)
-			mm->locked_vm += npage;
-
-		up_write(&mm->mmap_sem);
-	}
+	ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
+				  dma->lock_cap);
+	up_write(&mm->mmap_sem);
 
+out:
 	if (async)
 		mmput(mm);
 
include/linux/mm.h +19 -0
index 6b10c21630f5..7134e55ca23f 100644
@@ -1521,6 +1521,25 @@ static inline long get_user_pages_longterm(unsigned long start,
 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 			struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +22 -0
index 43a2984bccaa..552302665bc2 100644
@@ -6,6 +6,7 @@
 #include <linux/err.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
 #include <linux/security.h>
 #include <linux/swap.h>
@@ -346,6 +347,53 @@ int __weak get_user_pages_fast(unsigned long start,
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 
+/**
+ * __account_locked_vm - account locked pages to an mm's locked_vm
+ * @mm:          mm to account against, may be NULL
+ * @pages:       number of pages to account
+ * @inc:         %true if @pages should be considered positive, %false if not
+ * @task:        task used to check RLIMIT_MEMLOCK
+ * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
+ *
+ * Assumes @task and @mm are valid (i.e. at least one reference on each), and
+ * that mmap_sem is held as writer.
+ *
+ * Return:
+ * * 0       on success
+ * * 0       if @mm is NULL (can happen for example if the task is exiting)
+ * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
+ */
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim)
+{
+	unsigned long locked_vm, limit;
+	int ret = 0;

mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 380

Quoted 10 lines
mm/util.c +10 -0
+
+	locked_vm = mm->locked_vm;
+	if (inc) {
+		if (!bypass_rlim) {
+			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+			if (locked_vm + pages > limit) {
+				ret = -ENOMEM;
+				goto out;
+			}
+		}

Nit:

if (!ret)

and then you don't need "goto out".

mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 384

Quoted 4 lines
mm/util.c +4 -0
+		mm->locked_vm = locked_vm + pages;
+	} else {
+		WARN_ON_ONCE(pages > locked_vm);
+		mm->locked_vm = locked_vm - pages;

Can go negative here. Not a huge deal but inaccurate imo.

Quoted 18 lines
mm/util.c +12 -0
+	}
+
+out:
+	pr_debug("%s: [%d] %c%lu %lu/%lu%s\n", __func__, task->pid,
+		 (inc) ? '+' : '-', pages << PAGE_SHIFT,
+		 locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
+		 ret ? " - exceeded" : "");
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__account_locked_vm);
+
 unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
 	unsigned long len, unsigned long prot,
 	unsigned long flag, unsigned long pgoff)

base-commit: 37624b58542fb9f2d9a70e6ea006ef8a5f66c30b

Signature · 1 line
Alexey
#4 lore raw
Quoted 7 lines · Alexey Kardashevskiy
Quoted 3 lines · Daniel Jordan

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.

And I rather liked that prints were different and tell precisely which one of three each printk is.

I'm not following. One of three...callsites? But there were five callsites.

Anyway, I added a _RET_IP_ to the debug print so you can differentiate.

Quoted 3 lines

I commented below but in general this seems working.

Thanks! And for the review as well.

Quoted 8 lines
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +0 -0
index 6b64e45a5269..d39a1b830d82 100644
@@ -34,49 +35,13 @@
 static void tce_iommu_detach_group(void *iommu_data,
 		struct iommu_group *iommu_group);
 
Quoted 13 lines

drivers/vfio/vfio_iommu_spapr_tce.c line 42

Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2 -3
-static long try_increment_locked_vm(struct mm_struct *mm, long npages)
+static int tce_account_locked_vm(struct mm_struct *mm, unsigned long npages,
+				 bool inc)
 {
-	long ret = 0, locked, lock_limit;
-
 	if (WARN_ON_ONCE(!mm))
 		return -EPERM;

If this WARN_ON is the only reason for having tce_account_locked_vm() instead of calling account_locked_vm() directly, you can then ditch the check as I have never ever seen this triggered.

Great, will do.

Quoted 17 lines
Quoted 17 lines
drivers/vfio/vfio_iommu_type1.c +0 -9
index d0f731c9920a..15ac76171ccd 100644
@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 		return -ESRCH; /* process exited */
 
 	ret = down_write_killable(&mm->mmap_sem);
-	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
Quoted 8 lines

drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277

Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +2 -3
-					ret = -ENOMEM;
-			}
-		}
+	if (ret)
+		goto out;

A single "goto" to jump just 3 lines below seems unnecessary.

No strong preference here, I'll take out the goto.

Quoted 18 lines
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +5 -0
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim)
+{
+	unsigned long locked_vm, limit;
+	int ret = 0;

drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)

Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +10 -0
+
+	locked_vm = mm->locked_vm;
+	if (inc) {
+		if (!bypass_rlim) {
+			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+			if (locked_vm + pages > limit) {
+				ret = -ENOMEM;
+				goto out;
+			}
+		}

Nit:

Quoted 3 lines

if (!ret)

and then you don't need "goto out".

Ok, sure.

Quoted 7 lines

drivers/vfio/vfio_iommu_type1.c

Quoted 4 lines
drivers/vfio/vfio_iommu_type1.c +4 -0
+		mm->locked_vm = locked_vm + pages;
+	} else {
+		WARN_ON_ONCE(pages > locked_vm);
+		mm->locked_vm = locked_vm - pages;

Can go negative here. Not a huge deal but inaccurate imo.

I hear you, but setting a negative value to zero, as we had done previously, doesn't make much sense to me.

#5 lore raw
Quoted 10 lines · Daniel Jordan
Quoted 7 lines · Alexey Kardashevskiy
Quoted 3 lines · Daniel Jordan

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly.

And I rather liked that prints were different and tell precisely which one of three each printk is.

I'm not following. One of three...callsites? But there were five callsites.

Well, 3 of them are mine, I was referring to them :)

Quoted 1 line

Anyway, I added a _RET_IP_ to the debug print so you can differentiate.

I did not know that existed, cool!

Quoted 91 lines
Quoted 4 lines

I commented below but in general this seems working.

Thanks! And for the review as well.

Quoted 8 lines
Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +0 -0
index 6b64e45a5269..d39a1b830d82 100644
@@ -34,49 +35,13 @@
 static void tce_iommu_detach_group(void *iommu_data,
 		struct iommu_group *iommu_group);
 
Quoted 13 lines

drivers/vfio/vfio_iommu_spapr_tce.c line 42

Quoted 8 lines
drivers/vfio/vfio_iommu_spapr_tce.c +2 -3
-static long try_increment_locked_vm(struct mm_struct *mm, long npages)
+static int tce_account_locked_vm(struct mm_struct *mm, unsigned long npages,
+				 bool inc)
 {
-	long ret = 0, locked, lock_limit;
-
 	if (WARN_ON_ONCE(!mm))
 		return -EPERM;

If this WARN_ON is the only reason for having tce_account_locked_vm() instead of calling account_locked_vm() directly, you can then ditch the check as I have never ever seen this triggered.

Great, will do.

Quoted 17 lines
Quoted 17 lines
drivers/vfio/vfio_iommu_type1.c +0 -9
index d0f731c9920a..15ac76171ccd 100644
@@ -273,25 +273,14 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 		return -ESRCH; /* process exited */
 
 	ret = down_write_killable(&mm->mmap_sem);
-	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
Quoted 8 lines

drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) · line 277

Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +2 -3
-					ret = -ENOMEM;
-			}
-		}
+	if (ret)
+		goto out;

A single "goto" to jump just 3 lines below seems unnecessary.

No strong preference here, I'll take out the goto.

Quoted 18 lines
Quoted 5 lines
drivers/vfio/vfio_iommu_type1.c +5 -0
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim)
+{
+	unsigned long locked_vm, limit;
+	int ret = 0;

drivers/vfio/vfio_iommu_type1.c static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)

Quoted 10 lines
drivers/vfio/vfio_iommu_type1.c +10 -0
+
+	locked_vm = mm->locked_vm;
+	if (inc) {
+		if (!bypass_rlim) {
+			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+			if (locked_vm + pages > limit) {
+				ret = -ENOMEM;
+				goto out;
+			}
+		}

Nit:

Quoted 3 lines

if (!ret)

and then you don't need "goto out".

Ok, sure.

Quoted 7 lines

drivers/vfio/vfio_iommu_type1.c

Quoted 4 lines
drivers/vfio/vfio_iommu_type1.c +4 -0
+		mm->locked_vm = locked_vm + pages;
+	} else {
+		WARN_ON_ONCE(pages > locked_vm);
+		mm->locked_vm = locked_vm - pages;

Can go negative here. Not a huge deal but inaccurate imo.

Quoted 2 lines

I hear you, but setting a negative value to zero, as we had done previously, doesn't make much sense to me.

Ok then. I have not seen these WARN_ON for a very long time anyway.

Signature · 1 line
Alexey
#6 lore raw

PATCH v2 1/1 mm: add account_locked_vm utility function

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

Cc Alan Tull <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

---

Against v5.2-rc1.

v2:
- applied review comments from Alexey
- added _RET_IP_ to debug print to disambiguate callers

 arch/powerpc/kvm/book3s_64_vio.c     | 44 +++--------------------
 arch/powerpc/mm/book3s64/iommu_api.c | 41 +++------------------
 drivers/fpga/dfl-afu-dma-region.c    | 53 +++------------------------
 drivers/vfio/vfio_iommu_spapr_tce.c  | 54 +++-------------------------
 drivers/vfio/vfio_iommu_type1.c      | 17 ++-------
 include/linux/mm.h                   | 19 ++++++++++
 mm/util.c                            | 46 ++++++++++++++++++++++++
 7 files changed, 84 insertions(+), 190 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
index 66270e07449a..768b645c7edf 100644
@@ -30,6 +30,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/iommu.h>
 #include <linux/file.h>
+#include <linux/mm.h>
 
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
 	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
 }
 
-static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
-{
-	long ret = 0;
-
-	if (!current || !current->mm)
-		return ret; /* process exited */
-
-	down_write(&current->mm->mmap_sem);
-
-	if (inc) {
-		unsigned long locked, lock_limit;
-
-		locked = current->mm->locked_vm + stt_pages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += stt_pages;
-	} else {
-		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
-			stt_pages = current->mm->locked_vm;
-
-		current->mm->locked_vm -= stt_pages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
-			inc ? '+' : '-',
-			stt_pages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
 {
 	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
 
 	kvm_put_kvm(stt->kvm);
 
-	kvmppc_account_memlimit(
+	account_locked_vm(current->mm,
 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
 	call_rcu(&stt->rcu, release_spapr_tce_table);
 
@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 		return -EINVAL;
 
 	npages = kvmppc_tce_pages(size);
-	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
+	ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
 	if (ret)
 		return ret;
 
@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 
 	kfree(stt);
  fail_acct:
-	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
+	account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
 	return ret;
 }
 
arch/powerpc/mm/book3s64/iommu_api.c +4 -37

71 lines not on this page · raw message

drivers/fpga/dfl-afu-dma-region.c +4 -49

83 lines not on this page · raw message

drivers/vfio/vfio_iommu_spapr_tce.c +5 -49

90 lines not on this page · raw message

drivers/vfio/vfio_iommu_type1.c +2 -15
index 3ddc375e7063..bf449ace1676 100644
@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 
 	ret = down_write_killable(&mm->mmap_sem);
 	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
-					ret = -ENOMEM;
-			}
-		}
-
-		if (!ret)
-			mm->locked_vm += npage;
-
+		ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
+					  dma->lock_cap);
 		up_write(&mm->mmap_sem);
 	}
 
include/linux/mm.h +19 -0
index 0e8834ac32b7..72c1034d2ec7 100644
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +46 -0

58 lines not on this page · raw message

base-commit: a188339ca5a396acc588e5851ed7e19f66b0ebd9

Signature · 1 line
2.21.0
maintainer
#7 lore raw
Quoted 25 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

...

include/linux/mm.h +8 -0
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+

include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584

Quoted 10 lines
include/linux/mm.h +10 -0
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}

That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling drivers/vfio/vfio_iommu_type1.c to suit. I wonder why it does down_write_killable and whether it really needs to...

#8 lore raw
Quoted 34 lines · Andrew Morton
Quoted 32 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

...

include/linux/mm.h +15 -0
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
Quoted 8 lines

include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584

Quoted 3 lines
include/linux/mm.h +3 -0
+
+	return ret;
+}

That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling drivers/vfio/vfio_iommu_type1.c to suit. I wonder why it does down_write_killable and whether it really needs to...

Sure, I can uninline it. vfio changelogs don't show a particular reason for _killable[1]. Maybe Alex has something to add. Otherwise I'll respin without it since the simplification seems worth removing _killable.

[1] 0cfef2b7410b ("vfio/type1: Remove locked page accounting workqueue")

#9 lore raw
Quoted 44 lines · On Tue, 28 May 2019 11:04:24 -0400 Daniel Jordan <[email protected]>…
Quoted 34 lines · Andrew Morton
Quoted 32 lines · On Fri, 24 May 2019 13:50:45 -0400 Daniel Jordan <[email protected]>…

locked_vm accounting is done roughly the same way in five places, so unify them in a helper. Standardize the debug prints, which vary slightly, but include the helper's caller to disambiguate between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

...

include/linux/mm.h +15 -0
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
Quoted 8 lines

include/linux/mm.h long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, · line 1584

Quoted 3 lines
include/linux/mm.h +3 -0
+
+	return ret;
+}  

That's quite a mouthful for an inlined function. How about uninlining the whole thing and fiddling drivers/vfio/vfio_iommu_type1.c to suit. I wonder why it does down_write_killable and whether it really needs to...

Quoted 5 lines

Sure, I can uninline it. vfio changelogs don't show a particular reason for _killable[1]. Maybe Alex has something to add. Otherwise I'll respin without it since the simplification seems worth removing _killable.

[1] 0cfef2b7410b ("vfio/type1: Remove locked page accounting workqueue")

A userspace vfio driver maps DMA via an ioctl through this path, so I believe I used killable here just to be friendly that it could be interrupted and we could fall out with an errno if it were stuck here. No harm, no foul, the user's mapping is aborted and unwound. If we're deadlocked or seriously contended on mmap_sem, maybe we're already in trouble, but it seemed like a valid and low hanging use case for killable. Thanks,

Alex

#10 lore raw

PATCH v3 1/1 mm: add account_locked_vm utility function

locked_vm accounting is done roughly the same way in five places, so unify them in a helper.

Include the helper's caller in the debug print to distinguish between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

Cc Alan Tull <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Ira Weiny <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

---
v3:
- uninline account_locked_vm (Andrew)
- fix doc comment (Ira)
- retain down_write_killable in vfio type1 (Alex)
- leave Alexey's T-b since the code is the same aside from uninlining
- sanity tested with vfio type1, sanity-built on ppc

 arch/powerpc/kvm/book3s_64_vio.c     | 44 ++--------------
 arch/powerpc/mm/book3s64/iommu_api.c | 41 ++-------------
 drivers/fpga/dfl-afu-dma-region.c    | 53 ++------------------
 drivers/vfio/vfio_iommu_spapr_tce.c  | 54 ++------------------
 drivers/vfio/vfio_iommu_type1.c      | 17 +------
 include/linux/mm.h                   |  4 ++
 mm/util.c                            | 75 ++++++++++++++++++++++++++++
 7 files changed, 98 insertions(+), 190 deletions(-)
arch/powerpc/kvm/book3s_64_vio.c +4 -40
index 66270e07449a..768b645c7edf 100644
@@ -30,6 +30,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/iommu.h>
 #include <linux/file.h>
+#include <linux/mm.h>
 
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
 	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
 }
 
-static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
-{
-	long ret = 0;
-
-	if (!current || !current->mm)
-		return ret; /* process exited */
-
-	down_write(&current->mm->mmap_sem);
-
-	if (inc) {
-		unsigned long locked, lock_limit;
-
-		locked = current->mm->locked_vm + stt_pages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += stt_pages;
-	} else {
-		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
-			stt_pages = current->mm->locked_vm;
-
-		current->mm->locked_vm -= stt_pages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
-			inc ? '+' : '-',
-			stt_pages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
 {
 	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
 
 	kvm_put_kvm(stt->kvm);
 
-	kvmppc_account_memlimit(
+	account_locked_vm(current->mm,
 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
 	call_rcu(&stt->rcu, release_spapr_tce_table);
 
@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 		return -EINVAL;
 
 	npages = kvmppc_tce_pages(size);
-	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
+	ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
 	if (ret)
 		return ret;
 
@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 
 	kfree(stt);
  fail_acct:
-	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
+	account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
 	return ret;
 }
 
arch/powerpc/mm/book3s64/iommu_api.c +4 -37

71 lines not on this page · raw message

drivers/fpga/dfl-afu-dma-region.c +4 -49

83 lines not on this page · raw message

drivers/vfio/vfio_iommu_spapr_tce.c +5 -49

90 lines not on this page · raw message

drivers/vfio/vfio_iommu_type1.c +2 -15
index 3ddc375e7063..bf449ace1676 100644
@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 
 	ret = down_write_killable(&mm->mmap_sem);
 	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
-					ret = -ENOMEM;
-			}
-		}
-
-		if (!ret)
-			mm->locked_vm += npage;
-
+		ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
+					  dma->lock_cap);
 		up_write(&mm->mmap_sem);
 	}
 
include/linux/mm.h +4 -0
index 0e8834ac32b7..95510f6fad45 100644
@@ -1564,6 +1564,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +75 -0

87 lines not on this page · raw message

base-commit: cd6c84d8f0cdc911df435bb075ba22ce3c605b07

Signature · 1 line
2.21.0
Acked-by on v3 1/1
#11 lore raw
Quoted 41 lines · On Wed, 29 May 2019 16:50:19 -0400 Daniel Jordan <[email protected]>…

locked_vm accounting is done roughly the same way in five places, so unify them in a helper.

Include the helper's caller in the debug print to distinguish between callsites.

Error codes stay the same, so user-visible behavior does too. The one exception is that the -EPERM case in tce_account_locked_vm is removed because Alexey has never seen it triggered.

Cc Alan Tull <[email protected]>
Cc Alex Williamson <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Benjamin Herrenschmidt <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Christophe Leroy <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Ira Weiny <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Michael Ellerman <[email protected]>
Cc Moritz Fischer <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Steve Sistare <[email protected]>
Cc Wu Hao <[email protected]>

---
v3:
- uninline account_locked_vm (Andrew)
- fix doc comment (Ira)
- retain down_write_killable in vfio type1 (Alex)
- leave Alexey's T-b since the code is the same aside from uninlining
- sanity tested with vfio type1, sanity-built on ppc

Quoted 8 lines
 arch/powerpc/kvm/book3s_64_vio.c     | 44 ++--------------
 arch/powerpc/mm/book3s64/iommu_api.c | 41 ++-------------
 drivers/fpga/dfl-afu-dma-region.c    | 53 ++------------------
 drivers/vfio/vfio_iommu_spapr_tce.c  | 54 ++------------------
 drivers/vfio/vfio_iommu_type1.c      | 17 +------
 include/linux/mm.h                   |  4 ++
 mm/util.c                            | 75 ++++++++++++++++++++++++++++
 7 files changed, 98 insertions(+), 190 deletions(-)

I tend to prefer adding a negative rather than converting to absolute and passing a bool for inc/dec, but it all seems equivalent, so for vfio parts

Quoted 494 lines
arch/powerpc/kvm/book3s_64_vio.c +4 -40
index 66270e07449a..768b645c7edf 100644
@@ -30,6 +30,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/iommu.h>
 #include <linux/file.h>
+#include <linux/mm.h>
 
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
@@ -56,43 +57,6 @@ static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
 	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
 }
 
-static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
-{
-	long ret = 0;
-
-	if (!current || !current->mm)
-		return ret; /* process exited */
-
-	down_write(&current->mm->mmap_sem);
-
-	if (inc) {
-		unsigned long locked, lock_limit;
-
-		locked = current->mm->locked_vm + stt_pages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += stt_pages;
-	} else {
-		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
-			stt_pages = current->mm->locked_vm;
-
-		current->mm->locked_vm -= stt_pages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
-			inc ? '+' : '-',
-			stt_pages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
 {
 	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
@@ -302,7 +266,7 @@ static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
 
 	kvm_put_kvm(stt->kvm);
 
-	kvmppc_account_memlimit(
+	account_locked_vm(current->mm,
 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
 	call_rcu(&stt->rcu, release_spapr_tce_table);
 
@@ -327,7 +291,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 		return -EINVAL;
 
 	npages = kvmppc_tce_pages(size);
-	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
+	ret = account_locked_vm(current->mm, kvmppc_stt_pages(npages), true);
 	if (ret)
 		return ret;
 
@@ -373,7 +337,7 @@ long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
 
 	kfree(stt);
  fail_acct:
-	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
+	account_locked_vm(current->mm, kvmppc_stt_pages(npages), false);
 	return ret;
 }
 
arch/powerpc/mm/book3s64/iommu_api.c +4 -37
index 5c521f3924a5..18d22eec0ebd 100644
@@ -19,6 +19,7 @@
 #include <linux/hugetlb.h>
 #include <linux/swap.h>
 #include <linux/sizes.h>
+#include <linux/mm.h>
 #include <asm/mmu_context.h>
 #include <asm/pte-walk.h>
 #include <linux/mm_inline.h>
@@ -51,40 +52,6 @@ struct mm_iommu_table_group_mem_t {
 	u64 dev_hpa;		/* Device memory base address */
 };
 
-static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
-		unsigned long npages, bool incr)
-{
-	long ret = 0, locked, lock_limit;
-
-	if (!npages)
-		return 0;
-
-	down_write(&mm->mmap_sem);
-
-	if (incr) {
-		locked = mm->locked_vm + npages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			mm->locked_vm += npages;
-	} else {
-		if (WARN_ON_ONCE(npages > mm->locked_vm))
-			npages = mm->locked_vm;
-		mm->locked_vm -= npages;
-	}
-
-	pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
-			current ? current->pid : 0,
-			incr ? '+' : '-',
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK));
-	up_write(&mm->mmap_sem);
-
-	return ret;
-}
-
 bool mm_iommu_preregistered(struct mm_struct *mm)
 {
 	return !list_empty(&mm->context.iommu_group_mem_list);
@@ -101,7 +68,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
 	unsigned long entry, chunk;
 
 	if (dev_hpa = MM_IOMMU_TABLE_INVALID_HPA) {
-		ret = mm_iommu_adjust_locked_vm(mm, entries, true);
+		ret = account_locked_vm(mm, entries, true);
 		if (ret)
 			return ret;
 
@@ -216,7 +183,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
 	kfree(mem);
 
 unlock_exit:
-	mm_iommu_adjust_locked_vm(mm, locked_entries, false);
+	account_locked_vm(mm, locked_entries, false);
 
 	return ret;
 }
@@ -316,7 +283,7 @@ long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
 unlock_exit:
 	mutex_unlock(&mem_list_mutex);
 
-	mm_iommu_adjust_locked_vm(mm, unlock_entries, false);
+	account_locked_vm(mm, unlock_entries, false);
 
 	return ret;
 }
drivers/fpga/dfl-afu-dma-region.c +4 -49
index c438722bf4e1..0a532c602d8f 100644
@@ -12,6 +12,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/sched/signal.h>
 #include <linux/uaccess.h>
+#include <linux/mm.h>
 
 #include "dfl-afu.h"
 
@@ -31,52 +32,6 @@ void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
 	afu->dma_regions = RB_ROOT;
 }
 
-/**
- * afu_dma_adjust_locked_vm - adjust locked memory
- * @dev: port device
- * @npages: number of pages
- * @incr: increase or decrease locked memory
- *
- * Increase or decrease the locked memory size with npages input.
- *
- * Return 0 on success.
- * Return -ENOMEM if locked memory size is over the limit and no CAP_IPC_LOCK.
- */
-static int afu_dma_adjust_locked_vm(struct device *dev, long npages, bool incr)
-{
-	unsigned long locked, lock_limit;
-	int ret = 0;
-
-	/* the task is exiting. */
-	if (!current->mm)
-		return 0;
-
-	down_write(&current->mm->mmap_sem);
-
-	if (incr) {
-		locked = current->mm->locked_vm + npages;
-		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-			ret = -ENOMEM;
-		else
-			current->mm->locked_vm += npages;
-	} else {
-		if (WARN_ON_ONCE(npages > current->mm->locked_vm))
-			npages = current->mm->locked_vm;
-		current->mm->locked_vm -= npages;
-	}
-
-	dev_dbg(dev, "[%d] RLIMIT_MEMLOCK %c%ld %ld/%ld%s\n", current->pid,
-		incr ? '+' : '-', npages << PAGE_SHIFT,
-		current->mm->locked_vm << PAGE_SHIFT, rlimit(RLIMIT_MEMLOCK),
-		ret ? "- exceeded" : "");
-
-	up_write(&current->mm->mmap_sem);
-
-	return ret;
-}
-
 /**
  * afu_dma_pin_pages - pin pages of given dma memory region
  * @pdata: feature device platform data
@@ -92,7 +47,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
 	struct device *dev = &pdata->dev->dev;
 	int ret, pinned;
 
-	ret = afu_dma_adjust_locked_vm(dev, npages, true);
+	ret = account_locked_vm(current->mm, npages, true);
 	if (ret)
 		return ret;
 
@@ -121,7 +76,7 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
 free_pages:
 	kfree(region->pages);
 unlock_vm:
-	afu_dma_adjust_locked_vm(dev, npages, false);
+	account_locked_vm(current->mm, npages, false);
 	return ret;
 }
 
@@ -141,7 +96,7 @@ static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
 
 	put_all_pages(region->pages, npages);
 	kfree(region->pages);
-	afu_dma_adjust_locked_vm(dev, npages, false);
+	account_locked_vm(current->mm, npages, false);
 
 	dev_dbg(dev, "%ld pages unpinned\n", npages);
 }
drivers/vfio/vfio_iommu_spapr_tce.c +5 -49
index 40ddc0c5f677..d06e8e291924 100644
@@ -22,6 +22,7 @@
 #include <linux/vmalloc.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
+#include <linux/mm.h>
 
 #include <asm/iommu.h>
 #include <asm/tce.h>
@@ -34,51 +35,6 @@
 static void tce_iommu_detach_group(void *iommu_data,
 		struct iommu_group *iommu_group);
 
-static long try_increment_locked_vm(struct mm_struct *mm, long npages)
-{
-	long ret = 0, locked, lock_limit;
-
-	if (WARN_ON_ONCE(!mm))
-		return -EPERM;
-
-	if (!npages)
-		return 0;
-
-	down_write(&mm->mmap_sem);
-	locked = mm->locked_vm + npages;
-	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
-		ret = -ENOMEM;
-	else
-		mm->locked_vm += npages;
-
-	pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid,
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK),
-			ret ? " - exceeded" : "");
-
-	up_write(&mm->mmap_sem);
-
-	return ret;
-}
-
-static void decrement_locked_vm(struct mm_struct *mm, long npages)
-{
-	if (!mm || !npages)
-		return;
-
-	down_write(&mm->mmap_sem);
-	if (WARN_ON_ONCE(npages > mm->locked_vm))
-		npages = mm->locked_vm;
-	mm->locked_vm -= npages;
-	pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid,
-			npages << PAGE_SHIFT,
-			mm->locked_vm << PAGE_SHIFT,
-			rlimit(RLIMIT_MEMLOCK));
-	up_write(&mm->mmap_sem);
-}
-
 /*
  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
  *
@@ -336,7 +292,7 @@ static int tce_iommu_enable(struct tce_container *container)
 		return ret;
 
 	locked = table_group->tce32_size >> PAGE_SHIFT;
-	ret = try_increment_locked_vm(container->mm, locked);
+	ret = account_locked_vm(container->mm, locked, true);
 	if (ret)
 		return ret;
 
@@ -355,7 +311,7 @@ static void tce_iommu_disable(struct tce_container *container)
 	container->enabled = false;
 
 	BUG_ON(!container->mm);
-	decrement_locked_vm(container->mm, container->locked_pages);
+	account_locked_vm(container->mm, container->locked_pages, false);
 }
 
 static void *tce_iommu_open(unsigned long arg)
@@ -659,7 +615,7 @@ static long tce_iommu_create_table(struct tce_container *container,
 	if (!table_size)
 		return -EINVAL;
 
-	ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT);
+	ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
 	if (ret)
 		return ret;
 
@@ -678,7 +634,7 @@ static void tce_iommu_free_table(struct tce_container *container,
 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
 
 	iommu_tce_table_put(tbl);
-	decrement_locked_vm(container->mm, pages);
+	account_locked_vm(container->mm, pages, false);
 }
 
 static long tce_iommu_create_window(struct tce_container *container,
drivers/vfio/vfio_iommu_type1.c +2 -15
index 3ddc375e7063..bf449ace1676 100644
@@ -275,21 +275,8 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 
 	ret = down_write_killable(&mm->mmap_sem);
 	if (!ret) {
-		if (npage > 0) {
-			if (!dma->lock_cap) {
-				unsigned long limit;
-
-				limit = task_rlimit(dma->task,
-						RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-				if (mm->locked_vm + npage > limit)
-					ret = -ENOMEM;
-			}
-		}
-
-		if (!ret)
-			mm->locked_vm += npage;
-
+		ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
+					  dma->lock_cap);
 		up_write(&mm->mmap_sem);
 	}
 
include/linux/mm.h +4 -0
index 0e8834ac32b7..95510f6fad45 100644
@@ -1564,6 +1564,10 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +75 -0
index 91682a2090ee..cbbcc035b12b 100644
@@ -7,6 +7,7 @@
 #include <linux/err.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
 #include <linux/security.h>
 #include <linux/swap.h>
@@ -347,6 +348,80 @@ int __weak get_user_pages_fast(unsigned long start,
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 
+/**
+ * __account_locked_vm - account locked pages to an mm's locked_vm
+ * @mm:          mm to account against
+ * @pages:       number of pages to account
+ * @inc:         %true if @pages should be considered positive, %false if not
+ * @task:        task used to check RLIMIT_MEMLOCK
+ * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
+ *
+ * Assumes @task and @mm are valid (i.e. at least one reference on each), and
+ * that mmap_sem is held as writer.
+ *
+ * Return:
+ * * 0       on success
+ * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
+ */
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim)
+{
+	unsigned long locked_vm, limit;
+	int ret = 0;
+
+	lockdep_assert_held_exclusive(&mm->mmap_sem);
+
+	locked_vm = mm->locked_vm;
+	if (inc) {
+		if (!bypass_rlim) {
+			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+			if (locked_vm + pages > limit)
+				ret = -ENOMEM;
+		}
+		if (!ret)
+			mm->locked_vm = locked_vm + pages;
+	} else {
+		WARN_ON_ONCE(pages > locked_vm);
+		mm->locked_vm = locked_vm - pages;
+	}
+
+	pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
+		 (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
+		 locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
+		 ret ? " - exceeded" : "");
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__account_locked_vm);
+
+/**
+ * account_locked_vm - account locked pages to an mm's locked_vm
+ * @mm:          mm to account against, may be NULL
+ * @pages:       number of pages to account
+ * @inc:         %true if @pages should be considered positive, %false if not
+ *
+ * Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
+ *
+ * Return:
+ * * 0       on success, or if mm is NULL
+ * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
+ */
+int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(account_locked_vm);
+
 unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
 	unsigned long len, unsigned long prot,
 	unsigned long flag, unsigned long pgoff)

base-commit: cd6c84d8f0cdc911df435bb075ba22ce3c605b07

#12 lore raw
In reply to Daniel Jordan

On Fri, May 24, 2019 at 01:50:45PM -0400, Daniel Jordan wrote:

[snip]

Quoted 43 lines
include/linux/mm.h +19 -0
index 0e8834ac32b7..72c1034d2ec7 100644
@@ -1564,6 +1564,25 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 int get_user_pages_fast(unsigned long start, int nr_pages,
 			unsigned int gup_flags, struct page **pages);
 
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim);
+
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages == 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+
 /* Container for pinned pfns / pages */
 struct frame_vector {
 	unsigned int nr_allocated;	/* Number of frames we have space for */
mm/util.c +1 -0
index e2e4f8c3fa12..bd3bdf16a084 100644
@@ -6,6 +6,7 @@
 #include <linux/err.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
 #include <linux/security.h>
 #include <linux/swap.h>

mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 352

Quoted 6 lines
mm/util.c +3 -0
@@ -346,6 +347,51 @@ int __weak get_user_pages_fast(unsigned long start,
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 
+/**
+ * __account_locked_vm - account locked pages to an mm's locked_vm
+ * @mm:          mm to account against, may be NULL

This kernel doc is wrong. You dereference mm straight away...

Quoted 10 lines
mm/util.c +10 -0
+ * @pages:       number of pages to account
+ * @inc:         %true if @pages should be considered positive, %false if not
+ * @task:        task used to check RLIMIT_MEMLOCK
+ * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
+ *
+ * Assumes @task and @mm are valid (i.e. at least one reference on each), and
+ * that mmap_sem is held as writer.
+ *
+ * Return:
+ * * 0       on success

mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 372

Quoted 10 lines
mm/util.c +10 -0
+ * * 0       if @mm is NULL (can happen for example if the task is exiting)
+ * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
+ */
+int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
+			struct task_struct *task, bool bypass_rlim)
+{
+	unsigned long locked_vm, limit;
+	int ret = 0;
+
+	locked_vm = mm->locked_vm;

here...

Perhaps the comment was meant to document account_locked_vm()? Or should the parameter checks be moved here?

Ira

Quoted 32 lines
mm/util.c +22 -0
+	if (inc) {
+		if (!bypass_rlim) {
+			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+			if (locked_vm + pages > limit)
+				ret = -ENOMEM;
+		}
+		if (!ret)
+			mm->locked_vm = locked_vm + pages;
+	} else {
+		WARN_ON_ONCE(pages > locked_vm);
+		mm->locked_vm = locked_vm - pages;
+	}
+
+	pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
+		 (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
+		 locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
+		 ret ? " - exceeded" : "");
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__account_locked_vm);
 
 
+
 unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
mm/util.c +0 -0
 	unsigned long len, unsigned long prot,
 	unsigned long flag, unsigned long pgoff)

base-commit: a188339ca5a396acc588e5851ed7e19f66b0ebd9

Signature · 1 line
2.21.0
#13 lore raw
Quoted 17 lines · Ira Weiny
Quoted 16 lines · Daniel Jordan
include/linux/mm.h +16 -0
+static inline int account_locked_vm(struct mm_struct *mm, unsigned long pages,
+				    bool inc)
+{
+	int ret;
+
+	if (pages = 0 || !mm)
+		return 0;
+
+	down_write(&mm->mmap_sem);
+	ret = __account_locked_vm(mm, pages, inc, current,
+				  capable(CAP_IPC_LOCK));
+	up_write(&mm->mmap_sem);
+
+	return ret;
+}
+

...snip...

Quoted 5 lines
Quoted 3 lines

+/**
+ * __account_locked_vm - account locked pages to an mm's locked_vm
+ * @mm: mm to account against, may be NULL

This kernel doc is wrong. You dereference mm straight away...

...snip...

Quoted 6 lines

mm/util.c int __weak get_user_pages_fast(unsigned long start, · line 372

Quoted 2 lines
mm/util.c +2 -0
+
+	locked_vm = mm->locked_vm;

here...

Perhaps the comment was meant to document account_locked_vm()?

Yes, the comment got out of sync when I moved the !mm check outside __account_locked_vm. Thanks for catching, will fix.