Add support for shared PTEs across processes

This patch adds basic page table sharing across tasks by making mshare syscall. It does this by creating a new mm_struct which hosts the shared vmas and page tables. This mm_struct is maintained as long as there is at least one task using the mshare'd range. It is cleaned up by the last mshare_unlink syscall.

mm/internal.h +2 -0

index cf50a471384e..68f82f0f8b66 100644
@@ -718,6 +718,8 @@ void vunmap_range_noflush(unsigned long start, unsigned long end);
 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
 		      unsigned long addr, int page_nid, int *flags);
 
+extern vm_fault_t find_shared_vma(struct vm_area_struct **vma,
+			unsigned long *addrp);
 static inline bool vma_is_shared(const struct vm_area_struct *vma)
 {
 	return vma->vm_flags & VM_SHARED_PT;

mm/memory.c +24 -0

index c125c4969913..c77c0d643ea8 100644
@@ -4776,6 +4776,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 			   unsigned int flags, struct pt_regs *regs)
 {
 	vm_fault_t ret;
+	bool shared = false;
 
 	__set_current_state(TASK_RUNNING);
 

@@ -4785,6 +4786,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 	/* do counter updates before entering really critical section. */
 	check_sync_rss_stat(current);
 
+	if (unlikely(vma_is_shared(vma))) {
+		ret = find_shared_vma(&vma, &address);
+		if (ret)
+			return ret;
+		if (!vma)
+			return VM_FAULT_SIGSEGV;
+		shared = true;
+	}
+
 	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
 					    flags & FAULT_FLAG_INSTRUCTION,
 					    flags & FAULT_FLAG_REMOTE))

@@ -4802,6 +4812,31 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 	else
 		ret = __handle_mm_fault(vma, address, flags);
 
+	/*
+	 * Release the read lock on shared VMA's parent mm unless
+	 * __handle_mm_fault released the lock already.
+	 * __handle_mm_fault sets VM_FAULT_RETRY in return value if
+	 * it released mmap lock. If lock was released, that implies
+	 * the lock would have been released on task's original mm if
+	 * this were not a shared PTE vma. To keep lock state consistent,
+	 * make sure to release the lock on task's original mm
+	 */
+	if (shared) {
+		int release_mmlock = 1;
+
+		if (!(ret & VM_FAULT_RETRY)) {
+			mmap_read_unlock(vma->vm_mm);

mm/mshare.c +9 -1

index cd2f7ad24d9d..d1896adcb00f 100644
@@ -17,18 +17,49 @@
 #include <linux/pseudo_fs.h>
 #include <linux/fileattr.h>
 #include <linux/refcount.h>
+#include <linux/mman.h>
 #include <linux/sched/mm.h>
 #include <uapi/linux/magic.h>
 #include <uapi/linux/limits.h>
 
 struct mshare_data {
-	struct mm_struct *mm;
+	struct mm_struct *mm, *host_mm;
 	mode_t mode;
 	refcount_t refcnt;
 };
 
 static struct super_block *msharefs_sb;
 
+/* Returns holding the host mm's lock for read.  Caller must release. */
+vm_fault_t
+find_shared_vma(struct vm_area_struct **vmap, unsigned long *addrp)
+{
+	struct vm_area_struct *vma, *guest = *vmap;
+	struct mshare_data *info = guest->vm_private_data;
+	struct mm_struct *host_mm = info->mm;

mm/mshare.c +42 -2

+	*addrp = host_addr;
+	mmap_read_lock(host_mm);
+	vma = find_vma(host_mm, host_addr);
+
+	/* XXX: expand stack? */
+	if (vma && vma->vm_start > host_addr)
+		vma = NULL;
+
+	*vmap = vma;
+	return 0;
+}
+
 static void
 msharefs_evict_inode(struct inode *inode)
 {

@@ -169,11 +200,13 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
 		unsigned long, len, int, oflag, mode_t, mode)
 {
 	struct mshare_data *info;
-	struct mm_struct *mm;
 	struct filename *fname = getname(name);
 	struct dentry *dentry;
 	struct inode *inode;
 	struct qstr namestr;
+	struct vm_area_struct *vma, *next, *new_vma;
+	struct mm_struct *new_mm;
+	unsigned long end;
 	int err = PTR_ERR(fname);
 
 	/*

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
 	if (IS_ERR(fname))
 		goto err_out;
 
+	end = addr + len;
+
 	/*
 	 * Does this mshare entry exist already? If it does, calling
 	 * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
 	inode_lock(d_inode(msharefs_sb->s_root));
 	dentry = d_lookup(msharefs_sb->s_root, &namestr);
 	if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+		inode = d_inode(dentry);
 		err = -EEXIST;
 		dput(dentry);
 		goto err_unlock_inode;
 	}
 
 	if (dentry) {
+		unsigned long mapaddr, prot = PROT_NONE;
+
 		inode = d_inode(dentry);
 		if (inode == NULL) {
+			mmap_write_unlock(current->mm);
 			err = -EINVAL;
 			goto err_out;
 		}
 		info = inode->i_private;
-		refcount_inc(&info->refcnt);
 		dput(dentry);
+
+		/*
+		 * Map in the address range as anonymous mappings
+		 */
+		oflag &= (O_RDONLY | O_WRONLY | O_RDWR);
+		if (oflag & O_RDONLY)
+			prot |= PROT_READ;
+		else if (oflag & O_WRONLY)
+			prot |= PROT_WRITE;
+		else if (oflag & O_RDWR)
+			prot |= (PROT_READ | PROT_WRITE);
+		mapaddr = vm_mmap(NULL, addr, len, prot,
+				MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, 0);
+		if (IS_ERR((void *)mapaddr)) {
+			err = -EINVAL;
+			goto err_out;
+		}
+
+		refcount_inc(&info->refcnt);
+
+		/*

mm/mshare.c +12 -7

 	} else {
-		mm = mm_alloc();
-		if (!mm)
+		unsigned long myaddr;
+		struct mm_struct *old_mm;
+
+		old_mm = current->mm;
+		new_mm = mm_alloc();
+		if (!new_mm)
 			return -ENOMEM;
 		info = kzalloc(sizeof(*info), GFP_KERNEL);
 		if (!info) {
 			err = -ENOMEM;
 			goto err_relmm;
 		}
-		mm->mmap_base = addr;
-		mm->task_size = addr + len;
-		if (!mm->task_size)
-			mm->task_size--;
-		info->mm = mm;
+		new_mm->mmap_base = addr;
+		new_mm->task_size = addr + len;
+		if (!new_mm->task_size)
+			new_mm->task_size--;
+		info->mm = new_mm;
+		info->host_mm = old_mm;

Quoted 5 lines · Khalid Aziz

This patch adds basic page table sharing across tasks by making mshare syscall. It does this by creating a new mm_struct which hosts the shared vmas and page tables. This mm_struct is maintained as long as there is at least one task using the mshare'd range. It is cleaned up by the last mshare_unlink syscall.

This sounds like a really good idea because it (in theory) totally separates the lifetime of the *source* of the page tables from the lifetime of the process that creates the mshare.

Quoted 65 lines

mm/internal.h +2 -0

index cf50a471384e..68f82f0f8b66 100644
@@ -718,6 +718,8 @@ void vunmap_range_noflush(unsigned long start, unsigned long end);
  int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
  		      unsigned long addr, int page_nid, int *flags);
 
+extern vm_fault_t find_shared_vma(struct vm_area_struct **vma,
+			unsigned long *addrp);
  static inline bool vma_is_shared(const struct vm_area_struct *vma)
  {
  	return vma->vm_flags & VM_SHARED_PT;

mm/memory.c +30 -0

index c125c4969913..c77c0d643ea8 100644
@@ -4776,6 +4776,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  			   unsigned int flags, struct pt_regs *regs)
  {
  	vm_fault_t ret;
+	bool shared = false;
 
  	__set_current_state(TASK_RUNNING);
 

@@ -4785,6 +4786,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  	/* do counter updates before entering really critical section. */
  	check_sync_rss_stat(current);
 
+	if (unlikely(vma_is_shared(vma))) {
+		ret = find_shared_vma(&vma, &address);
+		if (ret)
+			return ret;
+		if (!vma)
+			return VM_FAULT_SIGSEGV;
+		shared = true;
+	}
+
  	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
  					    flags & FAULT_FLAG_INSTRUCTION,
  					    flags & FAULT_FLAG_REMOTE))

@@ -4802,6 +4812,31 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  	else
  		ret = __handle_mm_fault(vma, address, flags);
 
+	/*
+	 * Release the read lock on shared VMA's parent mm unless
+	 * __handle_mm_fault released the lock already.
+	 * __handle_mm_fault sets VM_FAULT_RETRY in return value if
+	 * it released mmap lock. If lock was released, that implies
+	 * the lock would have been released on task's original mm if
+	 * this were not a shared PTE vma. To keep lock state consistent,
+	 * make sure to release the lock on task's original mm
+	 */
+	if (shared) {
+		int release_mmlock = 1;
+
+		if (!(ret & VM_FAULT_RETRY)) {
+			mmap_read_unlock(vma->vm_mm);
+			release_mmlock = 0;
+		} else if ((flags & FAULT_FLAG_ALLOW_RETRY) &&
+			(flags & FAULT_FLAG_RETRY_NOWAIT)) {
+			mmap_read_unlock(vma->vm_mm);
+			release_mmlock = 0;
+		}

mm/memory.c vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, · line 4838

Quoted 4 lines

mm/memory.c +4 -0

+
+		if (release_mmlock)
+			mmap_read_unlock(current->mm);
+	}

Are we guaranteed that current->mm == the original vma->vm_mm? Just a quick scan of handle_mm_fault() users shows a few suspect ones like hmm_range_fault() or iommu_v2.c::do_fault().

Quoted 35 lines

mm/mshare.c +14 -1

index cd2f7ad24d9d..d1896adcb00f 100644
@@ -17,18 +17,49 @@
  #include <linux/pseudo_fs.h>
  #include <linux/fileattr.h>
  #include <linux/refcount.h>
+#include <linux/mman.h>
  #include <linux/sched/mm.h>
  #include <uapi/linux/magic.h>
  #include <uapi/linux/limits.h>
 
  struct mshare_data {
-	struct mm_struct *mm;
+	struct mm_struct *mm, *host_mm;
  	mode_t mode;
  	refcount_t refcnt;
  };
 
  static struct super_block *msharefs_sb;
 
+/* Returns holding the host mm's lock for read.  Caller must release. */
+vm_fault_t
+find_shared_vma(struct vm_area_struct **vmap, unsigned long *addrp)
+{
+	struct vm_area_struct *vma, *guest = *vmap;
+	struct mshare_data *info = guest->vm_private_data;
+	struct mm_struct *host_mm = info->mm;
+	unsigned long host_addr;
+	pgd_t *pgd, *guest_pgd;
+
+	host_addr = *addrp - guest->vm_start + host_mm->mmap_base;
+	pgd = pgd_offset(host_mm, host_addr);

mm/mshare.c line 49

Quoted 5 lines

mm/mshare.c +5 -0

+	guest_pgd = pgd_offset(current->mm, *addrp);
+	if (!pgd_same(*guest_pgd, *pgd)) {
+		set_pgd(guest_pgd, *pgd);
+		return VM_FAULT_NOPAGE;
+	}

Is digging around in the other process's page tables OK without holding any locks?

Quoted 89 lines

mm/mshare.c +48 -2

+	*addrp = host_addr;
+	mmap_read_lock(host_mm);
+	vma = find_vma(host_mm, host_addr);
+
+	/* XXX: expand stack? */
+	if (vma && vma->vm_start > host_addr)
+		vma = NULL;
+
+	*vmap = vma;
+	return 0;
+}
+
  static void
  msharefs_evict_inode(struct inode *inode)
  {

@@ -169,11 +200,13 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
  		unsigned long, len, int, oflag, mode_t, mode)
  {
  	struct mshare_data *info;
-	struct mm_struct *mm;
  	struct filename *fname = getname(name);
  	struct dentry *dentry;
  	struct inode *inode;
  	struct qstr namestr;
+	struct vm_area_struct *vma, *next, *new_vma;
+	struct mm_struct *new_mm;
+	unsigned long end;
  	int err = PTR_ERR(fname);
 
  	/*

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
  	if (IS_ERR(fname))
  		goto err_out;
 
+	end = addr + len;
+
  	/*
  	 * Does this mshare entry exist already? If it does, calling
  	 * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
  	inode_lock(d_inode(msharefs_sb->s_root));
  	dentry = d_lookup(msharefs_sb->s_root, &namestr);
  	if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+		inode = d_inode(dentry);
  		err = -EEXIST;
  		dput(dentry);
  		goto err_unlock_inode;
  	}
 
  	if (dentry) {
+		unsigned long mapaddr, prot = PROT_NONE;
+
  		inode = d_inode(dentry);
  		if (inode == NULL) {
+			mmap_write_unlock(current->mm);
  			err = -EINVAL;
  			goto err_out;
  		}
  		info = inode->i_private;
-		refcount_inc(&info->refcnt);
  		dput(dentry);
+
+		/*
+		 * Map in the address range as anonymous mappings
+		 */
+		oflag &= (O_RDONLY | O_WRONLY | O_RDWR);
+		if (oflag & O_RDONLY)
+			prot |= PROT_READ;
+		else if (oflag & O_WRONLY)
+			prot |= PROT_WRITE;
+		else if (oflag & O_RDWR)
+			prot |= (PROT_READ | PROT_WRITE);
+		mapaddr = vm_mmap(NULL, addr, len, prot,
+				MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, 0);
+		if (IS_ERR((void *)mapaddr)) {
+			err = -EINVAL;
+			goto err_out;
+		}
+
+		refcount_inc(&info->refcnt);
+
+		/*
+		 * Now that we have mmap'd the mshare'd range, update vma
+		 * flags and vm_mm pointer for this mshare'd range.
+		 */
+		mmap_write_lock(current->mm);
+		vma = find_vma(current->mm, addr);
+		if (vma && vma->vm_start < addr) {

mm/mshare.c SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr, · line 276

Quoted 4 lines

mm/mshare.c +4 -0

+			mmap_write_unlock(current->mm);
+			err = -EINVAL;
+			goto err_out;
+		}

How do you know that this is the same anonymous VMA that you set up above? Couldn't it have been unmapped and remapped to be something random before the mmap_write_lock() is reacquired?

mm/mshare.c SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr, · line 298

Quoted 8 lines

mm/mshare.c +7 -0

+		while (vma && vma->vm_start < (addr + len)) {
+			vma->vm_private_data = info;
+			vma->vm_mm = info->mm;
+			vma->vm_flags |= VM_SHARED_PT;
+			next = vma->vm_next;
+			vma = next;
+		}

This vma is still in the mm->mm_rb tree, right? I'm kinda surprised that it's OK to have a VMA in mm->vm_rb have vma->vm_mm!=mm.

Quoted 35 lines

mm/mshare.c +18 -7

  	} else {
-		mm = mm_alloc();
-		if (!mm)
+		unsigned long myaddr;
+		struct mm_struct *old_mm;
+
+		old_mm = current->mm;
+		new_mm = mm_alloc();
+		if (!new_mm)
  			return -ENOMEM;
  		info = kzalloc(sizeof(*info), GFP_KERNEL);
  		if (!info) {
  			err = -ENOMEM;
  			goto err_relmm;
  		}
-		mm->mmap_base = addr;
-		mm->task_size = addr + len;
-		if (!mm->task_size)
-			mm->task_size--;
-		info->mm = mm;
+		new_mm->mmap_base = addr;
+		new_mm->task_size = addr + len;
+		if (!new_mm->task_size)
+			new_mm->task_size--;
+		info->mm = new_mm;
+		info->host_mm = old_mm;
  		info->mode = mode;
  		refcount_set(&info->refcnt, 1);
+
+		/*
+		 * VMAs for this address range may or may not exist.
+		 * If VMAs exist, they should be marked as shared at
+		 * this point and page table info should be copied
+		 * over to newly created mm_struct. TODO: If VMAs do not

mm/mshare.c SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr, · line 326

Quoted 2 lines

mm/mshare.c +2 -0

+		 * exist, create them and mark them as shared.
+		 */

At this point, there are just too many TODO's in this series to look at it seriously. I think what you have here is an entertaining proof-of-concept, but it's looking to me to be obviously still RFC quality. Do you seriously think anyone could even *think* about applying this series at this point?

This patch adds basic page table sharing across tasks by making mshare syscall. It does this by creating a new mm_struct which hosts the shared vmas and page tables. This mm_struct is maintained as long as there is at least one task using the mshare'd range. It is cleaned up by the last mshare_unlink syscall.

Signed-off-by Khalid Aziz <[email protected]>

Signed-off-by Matthew Wilcox (Oracle) <[email protected]>

---
 mm/internal.h |   2 +
 mm/memory.c   |  35 ++++++++++
 mm/mshare.c   | 190 ++++++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 214 insertions(+), 13 deletions(-)

mm/internal.h +2 -0

index cf50a471384e..68f82f0f8b66 100644
@@ -718,6 +718,8 @@ void vunmap_range_noflush(unsigned long start, unsigned long end);
 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
                      unsigned long addr, int page_nid, int *flags);
 
+extern vm_fault_t find_shared_vma(struct vm_area_struct **vma,
+                       unsigned long *addrp);
 static inline bool vma_is_shared(const struct vm_area_struct *vma)
 {
        return vma->vm_flags & VM_SHARED_PT;

mm/memory.c +35 -0

index c125c4969913..c77c0d643ea8 100644
@@ -4776,6 +4776,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
                           unsigned int flags, struct pt_regs *regs)
 {
        vm_fault_t ret;
+       bool shared = false;
 
        __set_current_state(TASK_RUNNING);
 

@@ -4785,6 +4786,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
        /* do counter updates before entering really critical section. */
        check_sync_rss_stat(current);
 
+       if (unlikely(vma_is_shared(vma))) {
+               ret = find_shared_vma(&vma, &address);
+               if (ret)
+                       return ret;
+               if (!vma)
+                       return VM_FAULT_SIGSEGV;
+               shared = true;
+       }
+
        if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
                                            flags & FAULT_FLAG_INSTRUCTION,
                                            flags & FAULT_FLAG_REMOTE))

@@ -4802,6 +4812,31 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
        else
                ret = __handle_mm_fault(vma, address, flags);
 
+       /*
+        * Release the read lock on shared VMA's parent mm unless
+        * __handle_mm_fault released the lock already.
+        * __handle_mm_fault sets VM_FAULT_RETRY in return value if
+        * it released mmap lock. If lock was released, that implies
+        * the lock would have been released on task's original mm if
+        * this were not a shared PTE vma. To keep lock state consistent,
+        * make sure to release the lock on task's original mm
+        */
+       if (shared) {
+               int release_mmlock = 1;
+
+               if (!(ret & VM_FAULT_RETRY)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               } else if ((flags & FAULT_FLAG_ALLOW_RETRY) &&
+                       (flags & FAULT_FLAG_RETRY_NOWAIT)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               }
+
+               if (release_mmlock)
+                       mmap_read_unlock(current->mm);
+       }
+
        if (flags & FAULT_FLAG_USER) {
                mem_cgroup_exit_user_fault();
                /*

mm/mshare.c +44 -3

index cd2f7ad24d9d..d1896adcb00f 100644
@@ -17,18 +17,49 @@
 #include <linux/pseudo_fs.h>
 #include <linux/fileattr.h>
 #include <linux/refcount.h>
+#include <linux/mman.h>
 #include <linux/sched/mm.h>
 #include <uapi/linux/magic.h>
 #include <uapi/linux/limits.h>
 
 struct mshare_data {
-       struct mm_struct *mm;
+       struct mm_struct *mm, *host_mm;
        mode_t mode;
        refcount_t refcnt;
 };
 
 static struct super_block *msharefs_sb;
 
+/* Returns holding the host mm's lock for read.  Caller must release. */
+vm_fault_t
+find_shared_vma(struct vm_area_struct **vmap, unsigned long *addrp)
+{
+       struct vm_area_struct *vma, *guest = *vmap;
+       struct mshare_data *info = guest->vm_private_data;
+       struct mm_struct *host_mm = info->mm;
+       unsigned long host_addr;
+       pgd_t *pgd, *guest_pgd;
+
+       host_addr = *addrp - guest->vm_start + host_mm->mmap_base;
+       pgd = pgd_offset(host_mm, host_addr);
+       guest_pgd = pgd_offset(current->mm, *addrp);
+       if (!pgd_same(*guest_pgd, *pgd)) {
+               set_pgd(guest_pgd, *pgd);
+               return VM_FAULT_NOPAGE;
+       }
+
+       *addrp = host_addr;
+       mmap_read_lock(host_mm);
+       vma = find_vma(host_mm, host_addr);
+
+       /* XXX: expand stack? */
+       if (vma && vma->vm_start > host_addr)
+               vma = NULL;
+
+       *vmap = vma;
+       return 0;
+}
+
 static void
 msharefs_evict_inode(struct inode *inode)
 {

@@ -169,11 +200,13 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
                unsigned long, len, int, oflag, mode_t, mode)
 {
        struct mshare_data *info;
-       struct mm_struct *mm;
        struct filename *fname = getname(name);
        struct dentry *dentry;
        struct inode *inode;
        struct qstr namestr;
+       struct vm_area_struct *vma, *next, *new_vma;
+       struct mm_struct *new_mm;
+       unsigned long end;
        int err = PTR_ERR(fname);
 
        /*

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
        if (IS_ERR(fname))
                goto err_out;
 
+       end = addr + len;
+
        /*
         * Does this mshare entry exist already? If it does, calling
         * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
        inode_lock(d_inode(msharefs_sb->s_root));
        dentry = d_lookup(msharefs_sb->s_root, &namestr);
        if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+               inode = d_inode(dentry);
                err = -EEXIST;
                dput(dentry);
                goto err_unlock_inode;
        }
 
        if (dentry) {
+               unsigned long mapaddr, prot = PROT_NONE;
+
                inode = d_inode(dentry);
                if (inode == NULL) {
+                       mmap_write_unlock(current->mm);
                        err = -EINVAL;
                        goto err_out;
                }
                info = inode->i_private;
-               refcount_inc(&info->refcnt);
                dput(dentry);
+
+               /*
+                * Map in the address range as anonymous mappings

mm/mshare.c +123 -11

+               if (IS_ERR((void *)mapaddr)) {
+                       err = -EINVAL;
+                       goto err_out;
+               }
+
+               refcount_inc(&info->refcnt);
+
+               /*
+                * Now that we have mmap'd the mshare'd range, update vma
+                * flags and vm_mm pointer for this mshare'd range.
+                */
+               mmap_write_lock(current->mm);
+               vma = find_vma(current->mm, addr);
+               if (vma && vma->vm_start < addr) {
+                       mmap_write_unlock(current->mm);
+                       err = -EINVAL;
+                       goto err_out;
+               }
+
+               while (vma && vma->vm_start < (addr + len)) {
+                       vma->vm_private_data = info;
+                       vma->vm_mm = info->mm;
+                       vma->vm_flags |= VM_SHARED_PT;
+                       next = vma->vm_next;
+                       vma = next;
+               }
        } else {
-               mm = mm_alloc();
-               if (!mm)
+               unsigned long myaddr;
+               struct mm_struct *old_mm;
+
+               old_mm = current->mm;
+               new_mm = mm_alloc();
+               if (!new_mm)
                        return -ENOMEM;
                info = kzalloc(sizeof(*info), GFP_KERNEL);
                if (!info) {
                        err = -ENOMEM;
                        goto err_relmm;
                }
-               mm->mmap_base = addr;
-               mm->task_size = addr + len;
-               if (!mm->task_size)
-                       mm->task_size--;
-               info->mm = mm;
+               new_mm->mmap_base = addr;
+               new_mm->task_size = addr + len;
+               if (!new_mm->task_size)
+                       new_mm->task_size--;
+               info->mm = new_mm;
+               info->host_mm = old_mm;
                info->mode = mode;
                refcount_set(&info->refcnt, 1);
+
+               /*
+                * VMAs for this address range may or may not exist.
+                * If VMAs exist, they should be marked as shared at
+                * this point and page table info should be copied
+                * over to newly created mm_struct. TODO: If VMAs do not
+                * exist, create them and mark them as shared.
+                */
+               mmap_write_lock(old_mm);
+               vma = find_vma_intersection(old_mm, addr, end);
+               if (!vma) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+               /*
+                * TODO: If the currently allocated VMA goes beyond the
+                * mshare'd range, this VMA needs to be split.
+                *
+                * Double check that source VMAs do not extend outside
+                * the range
+                */
+               vma = find_vma(old_mm, addr + len);
+               if (vma && vma->vm_start < (addr + len)) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+
+               vma = find_vma(old_mm, addr);
+               if (vma && vma->vm_start < addr) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+
+               mmap_write_lock(new_mm);
+               while (vma && vma->vm_start < (addr + len)) {
+                       /*
+                        * Copy this vma over to host mm
+                        */
+                       vma->vm_private_data = info;
+                       vma->vm_mm = new_mm;
+                       vma->vm_flags |= VM_SHARED_PT;
+                       new_vma = vm_area_dup(vma);
+                       if (!new_vma) {
+                               err = -ENOMEM;
+                               goto unlock;
+                       }
+                       err = insert_vm_struct(new_mm, new_vma);
+                       if (err)
+                               goto unlock;
+
+                       vma = vma->vm_next;
+               }
+               mmap_write_unlock(new_mm);
+
                err = mshare_file_create(fname, oflag, info);
                if (err)
-                       goto err_relinfo;
+                       goto unlock;
+
+               /*
+                * Copy over current PTEs
+                */
+               myaddr = addr;
+               while (myaddr < new_mm->task_size) {
+                       *pgd_offset(new_mm, myaddr) = *pgd_offset(old_mm, myaddr);
+                       myaddr += PGDIR_SIZE;
+               }
+               /*
+                * TODO: Free the corresponding page table in calling
+                * process
+                */
        }
 
+       mmap_write_unlock(current->mm);
        inode_unlock(d_inode(msharefs_sb->s_root));
        putname(fname);
        return 0;
 
-err_relinfo:
+unlock:
+       mmap_write_unlock(current->mm);
        kfree(info);
 err_relmm:
-       mmput(mm);
+       mmput(new_mm);
 err_unlock_inode:
        inode_unlock(d_inode(msharefs_sb->s_root));
 err_out:

@@ -294,11 +445,24 @@ SYSCALL_DEFINE1(mshare_unlink, const char *, name)
 
        /*
         * Is this the last reference?
+        * TODO: permission checks are needed before proceeding
         */
        if (refcount_dec_and_test(&info->refcnt)) {
                simple_unlink(d_inode(msharefs_sb->s_root), dentry);
                d_drop(dentry);
                d_delete(dentry);
+               /*
+                * TODO: Release all physical pages allocated for this
+                * mshare range and release associated page table. If
+                * the final unlink happens from the process that created
+                * mshare'd range, do we return page tables and pages to
+                * that process so the creating process can continue using
+                * the address range it had chosen to mshare at some
+                * point?
+                *
+                * TODO: unmap shared vmas from every task that is using
+                * this mshare'd range.
+                */
                mmput(info->mm);
                kfree(info);
        } else {
--

2.32.0

Quoted 44 lines · Khalid Aziz

mm/mshare.c +17 -1

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
         if (IS_ERR(fname))
                 goto err_out;
 
+       end = addr + len;
+
         /*
          * Does this mshare entry exist already? If it does, calling
          * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
         inode_lock(d_inode(msharefs_sb->s_root));
         dentry = d_lookup(msharefs_sb->s_root, &namestr);
         if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+               inode = d_inode(dentry);
                 err = -EEXIST;
                 dput(dentry);
                 goto err_unlock_inode;
         }
 
         if (dentry) {
+               unsigned long mapaddr, prot = PROT_NONE;
+
                 inode = d_inode(dentry);
                 if (inode == NULL) {
+                       mmap_write_unlock(current->mm);
                         err = -EINVAL;
                         goto err_out;
                 }
                 info = inode->i_private;
-               refcount_inc(&info->refcnt);
                 dput(dentry);
+
+               /*
+                * Map in the address range as anonymous mappings
+                */
+               oflag &= (O_RDONLY | O_WRONLY | O_RDWR);
+               if (oflag & O_RDONLY)
+                       prot |= PROT_READ;
+               else if (oflag & O_WRONLY)
+                       prot |= PROT_WRITE;
+               else if (oflag & O_RDWR)
+                       prot |= (PROT_READ | PROT_WRITE);

mm/mshare.c SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr, · line 272

Quoted 2 lines

mm/mshare.c +2 -0

+               mapaddr = vm_mmap(NULL, addr, len, prot,
+                               MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, 0);

From the perspective of hardware, do we have to use MAP_FIXED to make
sure those processes sharing PTE
use the same virtual address for the shared area? or actually we don't necessarily need it? as long as the
upper level pgtable entries point to the same lower level pgtable?

This patch adds basic page table sharing across tasks by making mshare syscall. It does this by creating a new mm_struct which hosts the shared vmas and page tables. This mm_struct is maintained as long as there is at least one task using the mshare'd range. It is cleaned up by the last mshare_unlink syscall.

Signed-off-by Khalid Aziz <[email protected]>

Signed-off-by Matthew Wilcox (Oracle) <[email protected]>

---
 mm/internal.h |   2 +
 mm/memory.c   |  35 ++++++++++
 mm/mshare.c   | 190 ++++++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 214 insertions(+), 13 deletions(-)

mm/internal.h +2 -0

index cf50a471384e..68f82f0f8b66 100644
@@ -718,6 +718,8 @@ void vunmap_range_noflush(unsigned long start, unsigned long end);
 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
                      unsigned long addr, int page_nid, int *flags);
 
+extern vm_fault_t find_shared_vma(struct vm_area_struct **vma,
+                       unsigned long *addrp);
 static inline bool vma_is_shared(const struct vm_area_struct *vma)
 {
        return vma->vm_flags & VM_SHARED_PT;

mm/memory.c +35 -0

index c125c4969913..c77c0d643ea8 100644
@@ -4776,6 +4776,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
                           unsigned int flags, struct pt_regs *regs)
 {
        vm_fault_t ret;
+       bool shared = false;
 
        __set_current_state(TASK_RUNNING);
 

@@ -4785,6 +4786,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
        /* do counter updates before entering really critical section. */
        check_sync_rss_stat(current);
 
+       if (unlikely(vma_is_shared(vma))) {
+               ret = find_shared_vma(&vma, &address);
+               if (ret)
+                       return ret;
+               if (!vma)
+                       return VM_FAULT_SIGSEGV;
+               shared = true;
+       }
+
        if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
                                            flags & FAULT_FLAG_INSTRUCTION,
                                            flags & FAULT_FLAG_REMOTE))

@@ -4802,6 +4812,31 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
        else
                ret = __handle_mm_fault(vma, address, flags);
 
+       /*
+        * Release the read lock on shared VMA's parent mm unless
+        * __handle_mm_fault released the lock already.
+        * __handle_mm_fault sets VM_FAULT_RETRY in return value if
+        * it released mmap lock. If lock was released, that implies
+        * the lock would have been released on task's original mm if
+        * this were not a shared PTE vma. To keep lock state consistent,
+        * make sure to release the lock on task's original mm
+        */
+       if (shared) {
+               int release_mmlock = 1;
+
+               if (!(ret & VM_FAULT_RETRY)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               } else if ((flags & FAULT_FLAG_ALLOW_RETRY) &&
+                       (flags & FAULT_FLAG_RETRY_NOWAIT)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               }
+
+               if (release_mmlock)
+                       mmap_read_unlock(current->mm);
+       }
+
        if (flags & FAULT_FLAG_USER) {
                mem_cgroup_exit_user_fault();
                /*

mm/mshare.c +58 -3

index cd2f7ad24d9d..d1896adcb00f 100644
@@ -17,18 +17,49 @@
 #include <linux/pseudo_fs.h>
 #include <linux/fileattr.h>
 #include <linux/refcount.h>
+#include <linux/mman.h>
 #include <linux/sched/mm.h>
 #include <uapi/linux/magic.h>
 #include <uapi/linux/limits.h>
 
 struct mshare_data {
-       struct mm_struct *mm;
+       struct mm_struct *mm, *host_mm;
        mode_t mode;
        refcount_t refcnt;
 };
 
 static struct super_block *msharefs_sb;
 
+/* Returns holding the host mm's lock for read.  Caller must release. */
+vm_fault_t
+find_shared_vma(struct vm_area_struct **vmap, unsigned long *addrp)
+{
+       struct vm_area_struct *vma, *guest = *vmap;
+       struct mshare_data *info = guest->vm_private_data;
+       struct mm_struct *host_mm = info->mm;
+       unsigned long host_addr;
+       pgd_t *pgd, *guest_pgd;
+
+       host_addr = *addrp - guest->vm_start + host_mm->mmap_base;
+       pgd = pgd_offset(host_mm, host_addr);
+       guest_pgd = pgd_offset(current->mm, *addrp);
+       if (!pgd_same(*guest_pgd, *pgd)) {
+               set_pgd(guest_pgd, *pgd);
+               return VM_FAULT_NOPAGE;
+       }
+
+       *addrp = host_addr;
+       mmap_read_lock(host_mm);
+       vma = find_vma(host_mm, host_addr);
+
+       /* XXX: expand stack? */
+       if (vma && vma->vm_start > host_addr)
+               vma = NULL;
+
+       *vmap = vma;
+       return 0;
+}
+
 static void
 msharefs_evict_inode(struct inode *inode)
 {

@@ -169,11 +200,13 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
                unsigned long, len, int, oflag, mode_t, mode)
 {
        struct mshare_data *info;
-       struct mm_struct *mm;
        struct filename *fname = getname(name);
        struct dentry *dentry;
        struct inode *inode;
        struct qstr namestr;
+       struct vm_area_struct *vma, *next, *new_vma;
+       struct mm_struct *new_mm;
+       unsigned long end;
        int err = PTR_ERR(fname);
 
        /*

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
        if (IS_ERR(fname))
                goto err_out;
 
+       end = addr + len;
+
        /*
         * Does this mshare entry exist already? If it does, calling
         * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
        inode_lock(d_inode(msharefs_sb->s_root));
        dentry = d_lookup(msharefs_sb->s_root, &namestr);
        if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+               inode = d_inode(dentry);
                err = -EEXIST;
                dput(dentry);
                goto err_unlock_inode;
        }
 
        if (dentry) {
+               unsigned long mapaddr, prot = PROT_NONE;
+
                inode = d_inode(dentry);
                if (inode == NULL) {
+                       mmap_write_unlock(current->mm);
                        err = -EINVAL;
                        goto err_out;
                }
                info = inode->i_private;
-               refcount_inc(&info->refcnt);
                dput(dentry);
+
+               /*
+                * Map in the address range as anonymous mappings
+                */
+               oflag &= (O_RDONLY | O_WRONLY | O_RDWR);
+               if (oflag & O_RDONLY)
+                       prot |= PROT_READ;
+               else if (oflag & O_WRONLY)
+                       prot |= PROT_WRITE;
+               else if (oflag & O_RDWR)
+                       prot |= (PROT_READ | PROT_WRITE);
+               mapaddr = vm_mmap(NULL, addr, len, prot,
+                               MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, 0);
+               if (IS_ERR((void *)mapaddr)) {
+                       err = -EINVAL;
+                       goto err_out;
+               }

mm/mshare.c +109 -11

+                       mmap_write_unlock(current->mm);
+                       err = -EINVAL;
+                       goto err_out;
+               }
+
+               while (vma && vma->vm_start < (addr + len)) {
+                       vma->vm_private_data = info;
+                       vma->vm_mm = info->mm;
+                       vma->vm_flags |= VM_SHARED_PT;
+                       next = vma->vm_next;
+                       vma = next;
+               }
        } else {
-               mm = mm_alloc();
-               if (!mm)
+               unsigned long myaddr;
+               struct mm_struct *old_mm;
+
+               old_mm = current->mm;
+               new_mm = mm_alloc();
+               if (!new_mm)
                        return -ENOMEM;
                info = kzalloc(sizeof(*info), GFP_KERNEL);
                if (!info) {
                        err = -ENOMEM;
                        goto err_relmm;
                }
-               mm->mmap_base = addr;
-               mm->task_size = addr + len;
-               if (!mm->task_size)
-                       mm->task_size--;
-               info->mm = mm;
+               new_mm->mmap_base = addr;
+               new_mm->task_size = addr + len;
+               if (!new_mm->task_size)
+                       new_mm->task_size--;
+               info->mm = new_mm;
+               info->host_mm = old_mm;
                info->mode = mode;
                refcount_set(&info->refcnt, 1);
+
+               /*
+                * VMAs for this address range may or may not exist.
+                * If VMAs exist, they should be marked as shared at
+                * this point and page table info should be copied
+                * over to newly created mm_struct. TODO: If VMAs do not
+                * exist, create them and mark them as shared.
+                */
+               mmap_write_lock(old_mm);
+               vma = find_vma_intersection(old_mm, addr, end);
+               if (!vma) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+               /*
+                * TODO: If the currently allocated VMA goes beyond the
+                * mshare'd range, this VMA needs to be split.
+                *
+                * Double check that source VMAs do not extend outside
+                * the range
+                */
+               vma = find_vma(old_mm, addr + len);
+               if (vma && vma->vm_start < (addr + len)) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+
+               vma = find_vma(old_mm, addr);
+               if (vma && vma->vm_start < addr) {
+                       err = -EINVAL;
+                       goto unlock;
+               }
+
+               mmap_write_lock(new_mm);
+               while (vma && vma->vm_start < (addr + len)) {
+                       /*
+                        * Copy this vma over to host mm
+                        */
+                       vma->vm_private_data = info;
+                       vma->vm_mm = new_mm;
+                       vma->vm_flags |= VM_SHARED_PT;
+                       new_vma = vm_area_dup(vma);
+                       if (!new_vma) {
+                               err = -ENOMEM;
+                               goto unlock;
+                       }
+                       err = insert_vm_struct(new_mm, new_vma);
+                       if (err)
+                               goto unlock;
+
+                       vma = vma->vm_next;
+               }
+               mmap_write_unlock(new_mm);
+
                err = mshare_file_create(fname, oflag, info);
                if (err)
-                       goto err_relinfo;
+                       goto unlock;
+
+               /*
+                * Copy over current PTEs
+                */
+               myaddr = addr;
+               while (myaddr < new_mm->task_size) {
+                       *pgd_offset(new_mm, myaddr) = *pgd_offset(old_mm, myaddr);
+                       myaddr += PGDIR_SIZE;
+               }
+               /*
+                * TODO: Free the corresponding page table in calling
+                * process
+                */
        }
 
+       mmap_write_unlock(current->mm);
        inode_unlock(d_inode(msharefs_sb->s_root));
        putname(fname);
        return 0;
 
-err_relinfo:
+unlock:
+       mmap_write_unlock(current->mm);
        kfree(info);
 err_relmm:
-       mmput(mm);
+       mmput(new_mm);
 err_unlock_inode:
        inode_unlock(d_inode(msharefs_sb->s_root));
 err_out:

@@ -294,11 +445,24 @@ SYSCALL_DEFINE1(mshare_unlink, const char *, name)
 
        /*
         * Is this the last reference?
+        * TODO: permission checks are needed before proceeding
         */
        if (refcount_dec_and_test(&info->refcnt)) {
                simple_unlink(d_inode(msharefs_sb->s_root), dentry);
                d_drop(dentry);
                d_delete(dentry);
+               /*
+                * TODO: Release all physical pages allocated for this
+                * mshare range and release associated page table. If
+                * the final unlink happens from the process that created
+                * mshare'd range, do we return page tables and pages to
+                * that process so the creating process can continue using
+                * the address range it had chosen to mshare at some
+                * point?
+                *
+                * TODO: unmap shared vmas from every task that is using
+                * this mshare'd range.
+                */
                mmput(info->mm);
                kfree(info);
        } else {
--

2.32.0

Quoted 212 lines · Khalid Aziz

This patch adds basic page table sharing across tasks by making mshare syscall. It does this by creating a new mm_struct which hosts the shared vmas and page tables. This mm_struct is maintained as long as there is at least one task using the mshare'd range. It is cleaned up by the last mshare_unlink syscall.

Signed-off-by Khalid Aziz <[email protected]>

Signed-off-by Matthew Wilcox (Oracle) <[email protected]>

---
  mm/internal.h |   2 +
  mm/memory.c   |  35 ++++++++++
  mm/mshare.c   | 190 ++++++++++++++++++++++++++++++++++++++++++++++----
  3 files changed, 214 insertions(+), 13 deletions(-)

mm/internal.h +2 -0

index cf50a471384e..68f82f0f8b66 100644
@@ -718,6 +718,8 @@ void vunmap_range_noflush(unsigned long start, unsigned long end);
  int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
                       unsigned long addr, int page_nid, int *flags);
 
+extern vm_fault_t find_shared_vma(struct vm_area_struct **vma,
+                       unsigned long *addrp);
  static inline bool vma_is_shared(const struct vm_area_struct *vma)
  {
         return vma->vm_flags & VM_SHARED_PT;

mm/memory.c +35 -0

index c125c4969913..c77c0d643ea8 100644
@@ -4776,6 +4776,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
                            unsigned int flags, struct pt_regs *regs)
  {
         vm_fault_t ret;
+       bool shared = false;
 
         __set_current_state(TASK_RUNNING);
 

@@ -4785,6 +4786,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
         /* do counter updates before entering really critical section. */
         check_sync_rss_stat(current);
 
+       if (unlikely(vma_is_shared(vma))) {
+               ret = find_shared_vma(&vma, &address);
+               if (ret)
+                       return ret;
+               if (!vma)
+                       return VM_FAULT_SIGSEGV;
+               shared = true;
+       }
+
         if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
                                             flags & FAULT_FLAG_INSTRUCTION,
                                             flags & FAULT_FLAG_REMOTE))

@@ -4802,6 +4812,31 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
         else
                 ret = __handle_mm_fault(vma, address, flags);
 
+       /*
+        * Release the read lock on shared VMA's parent mm unless
+        * __handle_mm_fault released the lock already.
+        * __handle_mm_fault sets VM_FAULT_RETRY in return value if
+        * it released mmap lock. If lock was released, that implies
+        * the lock would have been released on task's original mm if
+        * this were not a shared PTE vma. To keep lock state consistent,
+        * make sure to release the lock on task's original mm
+        */
+       if (shared) {
+               int release_mmlock = 1;
+
+               if (!(ret & VM_FAULT_RETRY)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               } else if ((flags & FAULT_FLAG_ALLOW_RETRY) &&
+                       (flags & FAULT_FLAG_RETRY_NOWAIT)) {
+                       mmap_read_unlock(vma->vm_mm);
+                       release_mmlock = 0;
+               }
+
+               if (release_mmlock)
+                       mmap_read_unlock(current->mm);
+       }
+
         if (flags & FAULT_FLAG_USER) {
                 mem_cgroup_exit_user_fault();
                 /*

mm/mshare.c +64 -3

index cd2f7ad24d9d..d1896adcb00f 100644
@@ -17,18 +17,49 @@
  #include <linux/pseudo_fs.h>
  #include <linux/fileattr.h>
  #include <linux/refcount.h>
+#include <linux/mman.h>
  #include <linux/sched/mm.h>
  #include <uapi/linux/magic.h>
  #include <uapi/linux/limits.h>
 
  struct mshare_data {
-       struct mm_struct *mm;
+       struct mm_struct *mm, *host_mm;
         mode_t mode;
         refcount_t refcnt;
  };
 
  static struct super_block *msharefs_sb;
 
+/* Returns holding the host mm's lock for read.  Caller must release. */
+vm_fault_t
+find_shared_vma(struct vm_area_struct **vmap, unsigned long *addrp)
+{
+       struct vm_area_struct *vma, *guest = *vmap;
+       struct mshare_data *info = guest->vm_private_data;
+       struct mm_struct *host_mm = info->mm;
+       unsigned long host_addr;
+       pgd_t *pgd, *guest_pgd;
+
+       host_addr = *addrp - guest->vm_start + host_mm->mmap_base;
+       pgd = pgd_offset(host_mm, host_addr);
+       guest_pgd = pgd_offset(current->mm, *addrp);
+       if (!pgd_same(*guest_pgd, *pgd)) {
+               set_pgd(guest_pgd, *pgd);
+               return VM_FAULT_NOPAGE;
+       }
+
+       *addrp = host_addr;
+       mmap_read_lock(host_mm);
+       vma = find_vma(host_mm, host_addr);
+
+       /* XXX: expand stack? */
+       if (vma && vma->vm_start > host_addr)
+               vma = NULL;
+
+       *vmap = vma;
+       return 0;
+}
+
  static void
  msharefs_evict_inode(struct inode *inode)
  {

@@ -169,11 +200,13 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
                 unsigned long, len, int, oflag, mode_t, mode)
  {
         struct mshare_data *info;
-       struct mm_struct *mm;
         struct filename *fname = getname(name);
         struct dentry *dentry;
         struct inode *inode;
         struct qstr namestr;
+       struct vm_area_struct *vma, *next, *new_vma;
+       struct mm_struct *new_mm;
+       unsigned long end;
         int err = PTR_ERR(fname);
 
         /*

@@ -193,6 +226,8 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
         if (IS_ERR(fname))
                 goto err_out;
 
+       end = addr + len;
+
         /*
          * Does this mshare entry exist already? If it does, calling
          * mshare with O_EXCL|O_CREAT is an error

@@ -205,49 +240,165 @@ SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr,
         inode_lock(d_inode(msharefs_sb->s_root));
         dentry = d_lookup(msharefs_sb->s_root, &namestr);
         if (dentry && (oflag & (O_EXCL|O_CREAT))) {
+               inode = d_inode(dentry);
                 err = -EEXIST;
                 dput(dentry);
                 goto err_unlock_inode;
         }
 
         if (dentry) {
+               unsigned long mapaddr, prot = PROT_NONE;
+
                 inode = d_inode(dentry);
                 if (inode == NULL) {
+                       mmap_write_unlock(current->mm);
                         err = -EINVAL;
                         goto err_out;
                 }
                 info = inode->i_private;
-               refcount_inc(&info->refcnt);
                 dput(dentry);
+
+               /*
+                * Map in the address range as anonymous mappings
+                */
+               oflag &= (O_RDONLY | O_WRONLY | O_RDWR);
+               if (oflag & O_RDONLY)
+                       prot |= PROT_READ;
+               else if (oflag & O_WRONLY)
+                       prot |= PROT_WRITE;
+               else if (oflag & O_RDWR)
+                       prot |= (PROT_READ | PROT_WRITE);
+               mapaddr = vm_mmap(NULL, addr, len, prot,
+                               MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, 0);
+               if (IS_ERR((void *)mapaddr)) {
+                       err = -EINVAL;
+                       goto err_out;
+               }
+
+               refcount_inc(&info->refcnt);
+
+               /*
+                * Now that we have mmap'd the mshare'd range, update vma
+                * flags and vm_mm pointer for this mshare'd range.

mm/mshare.c SYSCALL_DEFINE5(mshare, const char __user *, name, unsigned long, addr, · line 286

Quoted 4 lines

mm/mshare.c +4 -0

+                */
+               mmap_write_lock(current->mm);
+               vma = find_vma(current->mm, addr);
+               if (vma && vma->vm_start < addr) {

and I don't understand why "vma->vm_start < addr" can happen. does it mean we have given mshare() an address which is not aligned? then we should have return -EINVAL earlier?

mshare'd PTEs should not be removed when a task exits. These PTEs are removed when the last task sharing the PTEs exits. Add a check for shared PTEs and skip them.

Signed-off-by Khalid Aziz <[email protected]>

---
 mm/memory.c | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

mm/memory.c +12 -3

index c77c0d643ea8..e7c5bc6f8836 100644
@@ -419,16 +419,25 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
                } else {
                        /*
                         * Optimization: gather nearby vmas into one call down
+                        * as long as they all belong to the same mm (that
+                        * may not be the case if a vma is part of mshare'd
+                        * range
                         */
                        while (next && next->vm_start <= vma->vm_end + PMD_SIZE
-                              && !is_vm_hugetlb_page(next)) {
+                              && !is_vm_hugetlb_page(next)
+                              && vma->vm_mm == tlb->mm) {
                                vma = next;
                                next = vma->vm_next;
                                unlink_anon_vmas(vma);
                                unlink_file_vma(vma);
                        }
-                       free_pgd_range(tlb, addr, vma->vm_end,
-                               floor, next ? next->vm_start : ceiling);
+                       /*
+                        * Free pgd only if pgd is not allocated for an
+                        * mshare'd range
+                        */
+                       if (vma->vm_mm == tlb->mm)
+                               free_pgd_range(tlb, addr, vma->vm_end,
+                                       floor, next ? next->vm_start : ceiling);
                }
                vma = next;
        }

mm/memory.c +0 -1

        BUG_ON(addr >= end);
        tlb_start_vma(tlb, vma);
        pgd = pgd_offset(vma->vm_mm, addr);
--

2.32.0

Quoted 47 lines · Khalid Aziz

mshare'd PTEs should not be removed when a task exits. These PTEs are removed when the last task sharing the PTEs exits. Add a check for shared PTEs and skip them.

Signed-off-by Khalid Aziz <[email protected]>

---
  mm/memory.c | 22 +++++++++++++++++++---
  1 file changed, 19 insertions(+), 3 deletions(-)

mm/memory.c +12 -3

index c77c0d643ea8..e7c5bc6f8836 100644
@@ -419,16 +419,25 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 } else {
                         /*
                          * Optimization: gather nearby vmas into one call down
+                        * as long as they all belong to the same mm (that
+                        * may not be the case if a vma is part of mshare'd
+                        * range
                          */
                         while (next && next->vm_start <= vma->vm_end + PMD_SIZE
-                              && !is_vm_hugetlb_page(next)) {
+                              && !is_vm_hugetlb_page(next)
+                              && vma->vm_mm == tlb->mm) {
                                 vma = next;
                                 next = vma->vm_next;
                                 unlink_anon_vmas(vma);
                                 unlink_file_vma(vma);
                         }
-                       free_pgd_range(tlb, addr, vma->vm_end,
-                               floor, next ? next->vm_start : ceiling);
+                       /*
+                        * Free pgd only if pgd is not allocated for an
+                        * mshare'd range
+                        */
+                       if (vma->vm_mm == tlb->mm)
+                               free_pgd_range(tlb, addr, vma->vm_end,
+                                       floor, next ? next->vm_start : ceiling);
                 }
                 vma = next;
         }

@@ -1551,6 +1560,13 @@ void unmap_page_range(struct mmu_gather *tlb,
         pgd_t *pgd;
         unsigned long next;
 

mm/memory.c void unmap_page_range(struct mmu_gather *tlb, · line 1569

Quoted 7 lines

mm/memory.c +7 -0

+       /*
+        * If this is an mshare'd page, do not unmap it since it might
+        * still be in use.
+        */
+       if (vma->vm_mm != tlb->mm)
+               return;
+

expect unmap, have you ever tested reverse mapping in vmscan, especially folio_referenced()? are all vmas in those processes sharing page table still in the rmap of the shared page?
without shared PTE, if 1000 processes share one page, we are reading 1000 PTEs, with it, are we reading just one? or are we reading the same PTE 1000 times? Have you tested it?

Quoted 53 lines · Barry Song

Quoted 52 lines · Khalid Aziz

mshare'd PTEs should not be removed when a task exits. These PTEs are removed when the last task sharing the PTEs exits. Add a check for shared PTEs and skip them.

Signed-off-by Khalid Aziz <[email protected]>

---
  mm/memory.c | 22 +++++++++++++++++++---

  1 file changed, 19 insertions(+), 3 deletions(-)

mm/memory.c +1 -0

index c77c0d643ea8..e7c5bc6f8836 100644
@@ -419,16 +419,25 @@ void free_pgtables(struct mmu_gather *tlb, 
 struct vm_area_struct *vma,
                  } else {
                          /*
                           * Optimization: gather nearby vmas into one call down
+                        * as long as they all belong to the same mm 

(that
+                        * may not be the case if a vma is part of mshare'd
+                        * range
                          */
                         while (next && next->vm_start <= vma->vm_end
+ PMD_SIZE
-                              && !is_vm_hugetlb_page(next)) { +                              && !is_vm_hugetlb_page(next) +                              && vma->vm_mm == tlb->mm) { vma = next;
                                 next = vma->vm_next; unlink_anon_vmas(vma); unlink_file_vma(vma);
                         }
-                       free_pgd_range(tlb, addr, vma->vm_end, -                               floor, next ? next->vm_start : ceiling); +                       /*
+                        * Free pgd only if pgd is not allocated for an +                        * mshare'd range
+                        */
+                       if (vma->vm_mm == tlb->mm)
+                               free_pgd_range(tlb, addr, vma->vm_end, +                                       floor, next ? next->vm_start : ceiling);
                 }
                 vma = next;
         }

mm/memory.c +0 -0

@@ -1551,6 +1560,13 @@ void unmap_page_range(struct mmu_gather *tlb,
          pgd_t *pgd;
          unsigned long next;
 

Quoted 15 lines

mm/memory.c void unmap_page_range(struct mmu_gather *tlb, · line 1569

Quoted 7 lines

mm/memory.c +7 -0

+       /*
+        * If this is an mshare'd page, do not unmap it since it might
+        * still be in use.
+        */
+       if (vma->vm_mm != tlb->mm)
+               return;
+

expect unmap, have you ever tested reverse mapping in vmscan, especially folio_referenced()? are all vmas in those processes sharing page table still in the rmap of the shared page?
without shared PTE, if 1000 processes share one page, we are reading 1000 PTEs, with it, are we reading just one? or are we reading the same PTE 1000 times? Have you tested it?

We are treating mshared region same as threads sharing address space. There is one PTE that is being used by all processes and the VMA maintained in the separate mshare mm struct that also holds the shared PTE is the one that gets added to rmap. This is a different model with mshare in that it adds an mm struct that is separate from the mm structs of the processes that refer to the vma and pte in mshare mm struct. Do you see issues with rmap in this model?

Quoted 65 lines · Khalid Aziz

Quoted 48 lines · Barry Song

Quoted 47 lines · Khalid Aziz

mshare'd PTEs should not be removed when a task exits. These PTEs are removed when the last task sharing the PTEs exits. Add a check for shared PTEs and skip them.

Signed-off-by Khalid Aziz <[email protected]>

---
  mm/memory.c | 22 +++++++++++++++++++---

  1 file changed, 19 insertions(+), 3 deletions(-)

mm/memory.c +3 -0

index c77c0d643ea8..e7c5bc6f8836 100644
@@ -419,16 +419,25 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
                  } else {
                          /*
                           * Optimization: gather nearby vmas into one call down
+                        * as long as they all belong to the same mm (that
+                        * may not be the case if a vma is part of mshare'd
+                        * range
                           */

                         while (next && next->vm_start <= vma->vm_end + PMD_SIZE -                              && !is_vm_hugetlb_page(next)) {
+                              && !is_vm_hugetlb_page(next)
+                              && vma->vm_mm == tlb->mm) {
                                 vma = next;
                                 next = vma->vm_next; unlink_anon_vmas(vma); unlink_file_vma(vma);
                         }
-                       free_pgd_range(tlb, addr, vma->vm_end,
-                               floor, next ? next->vm_start : ceiling); +                       /*
+                        * Free pgd only if pgd is not allocated for an +                        * mshare'd range
+                        */
+                       if (vma->vm_mm == tlb->mm)
+                               free_pgd_range(tlb, addr, vma->vm_end, +                                       floor, next ? next->vm_start : ceiling); }
                 vma = next;
         }

mm/memory.c +0 -0

@@ -1551,6 +1560,13 @@ void unmap_page_range(struct mmu_gather *tlb,
          pgd_t *pgd;
          unsigned long next;
 

Quoted 15 lines

mm/memory.c void unmap_page_range(struct mmu_gather *tlb, · line 1569

Quoted 7 lines

mm/memory.c +7 -0

+       /*
+        * If this is an mshare'd page, do not unmap it since it might
+        * still be in use.
+        */
+       if (vma->vm_mm != tlb->mm)
+               return;
+

expect unmap, have you ever tested reverse mapping in vmscan, especially folio_referenced()? are all vmas in those processes sharing page table still in the rmap of the shared page?
without shared PTE, if 1000 processes share one page, we are reading 1000 PTEs, with it, are we reading just one? or are we reading the same PTE 1000 times? Have you tested it?

Quoted 1 line

We are treating mshared region same as threads sharing address space. There is one PTE that is being used by all processes and the VMA maintained in the separate mshare mm struct that also holds the shared PTE is the one that gets added to rmap. This is a different model with mshare in that it adds an mm struct that is separate from the mm structs of the processes that refer to the vma and pte in mshare mm struct. Do you see issues with rmap in this model?

I think this patch is actually the most interesting bit of the series by far.  Most of the rest is defining an API (which is important!) and figuring out semantics.  This patch changes something rather fundamental about how user address spaces work: what vmas live in them.  So let's figure out its effects.

I admit I'm rather puzzled about what vm_mm is for in the first place. In current kernels (without your patch), I think it's a pretty hard requirement for vm_mm to equal the mm for all vmas in an mm.  After a quick and incomplete survey, vm_mm seems to be mostly used as a somewhat lazy way to find the mm.  Let's see:

file_operations->mmap doesn't receive an mm_struct.  Instead it infers the mm from vm_mm.  (Why?  I don't know.)

Some walk_page_range users seem to dig the mm out of vm_mm instead of mm_walk.

Some manual address space walkers start with an mm, don't bother passing it around, and dig it back out of of vm_mm. For example, unuse_vma() and all its helpers.

The only real exception I've found so far is rmap: AFAICS (on quick inspection -- I could be wrong), rmap can map from a folio to a bunch of vmas, and the vmas' mms are not stored separately but instead determined by vm_mm.

Your patch makes me quite nervous.  You're potentially breaking any kernel code path that assumes that mms only contain vmas that have vm_mm == mm.  And you're potentially causing rmap to be quite confused.  I think that if you're going to take this approach, you need to clearly define the new semantics of vm_mm and audit or clean up every user of vm_mm in the tree.  This may be nontrivial (especially rmap), although a cleanup of everything else to stop using vm_mm might be valuable.

But I'm wondering if it would be better to attack this from a different direction.  Right now, there's a hardcoded assumption that an mm owns every page table it references.  That's really the thing you're changing.  To me, it seems that a magical vma that shares page tables should still be a vma that belongs to its mm_struct -- munmap() and potentialy other m***() operations should all work on it, existing find_vma() users should work, etc.

So maybe instead there should be new behavior (by a VM_ flag or otherwise) that indicates that a vma owns its PTEs.  It could even be a vm_operation, although if anyone ever wants regular file mappings to share PTEs, then a vm_operation doesn't really make sense.

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Another argument that MM developers find compelling is that we can reduce some of the complexity in hugetlbfs where it has the ability to share page tables between processes.

Quoted 3 lines · Matthew Wilcox

Another argument that MM developers find compelling is that we can reduce some of the complexity in hugetlbfs where it has the ability to share page tables between processes.

When could this complexity reduction actually happen in practice? Can this mshare thingy be somehow dropped in underneath the existing hugetlbfs implementation? Or would userspace need to change?

Quoted 8 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

All of this is true. However, I've found a lot of people don't see this as compelling. I've had more success explaining this from a different direction:

--- 8< ---

Linux supports processes which share all of their address space (threads) and processes that share none of their address space (tasks). We propose a useful intermediate model where two or more cooperating processes can choose to share portions of their address space with each other. The shared portion is referred to by a file descriptor which processes can choose to attach to their own address space.

Modifications to the shared region affect all processes sharing that region, just as changes by one thread affect all threads in a multithreaded program. This implies a certain level of trust between the different processes (ie malicious processes should not be allowed access to the mshared region).

--- 8< ---

Another argument that MM developers find compelling is that we can reduce some of the complexity in hugetlbfs where it has the ability to share page tables between processes.

One objection that was raised is that the mechanism for starting the shared region is a bit clunky. Did you investigate the proposed approach of creating an empty address space, attaching to it and using an fd-based mmap to modify its contents?

Quoted 9 lines

int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Can you explain why this is a syscall instead of being a library function which does

	int dirfd = open("/sys/fs/mshare");
	err = unlinkat(dirfd, name, 0);
	close(dirfd);
	return err;

Does msharefs support creating directories, so that we can use file permissions to limit who can see the sharable files? Or is it strictly a single-level-deep hierarchy?

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

Quoted 58 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

Quoted 5 lines

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

Please don't add system calls that take names.

Please just open objects on the filesystem and allow multi-instances of the filesystem. Otherwise someone is going to have to come along later and implement namespace magic to deal with your new system calls. You already have a filesystem all that is needed to avoid having to introduce namespace magic is to simply allow multiple instances of the filesystem to be mounted.

On that note. Since you have a filesystem, introduce a well known name for a directory and in that directory place all of the information and possibly control files for your filesystem. No need to for proc files and the like, and if at somepoint you have mount options that allow the information to be changed you can have different mounts with different values present.

This is must me. But I find it weird that you don't use mmap to place the shared area from the mshare fd into your address space.

I think I would do:

	// Establish the mshare region
	addr = mmap(NULL, PGDIR_SIZE, PROT_READ | PROT_WRITE,
			MAP_SHARED | MAP_MSHARE, msharefd, 0);

	// Remove the mshare region
         addr2 = mmap(addr, PGDIR_SIZE, PROT_NONE,
         		MAP_FIXED | MAP_MUNSHARE, msharefd, 0);

I could see a point of using separate system calls instead of adding MAP_SHARE and MAP_UNSHARE flags.

What are the locking implications of taking a page fault in the shared region?

Is it a noop or is it going to make some of the nasty locking we have in the kernel for things like truncates worse?

Eric

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

A file under /sys/fs/mshare can be opened and read from. A read from this file returns two long values - (1) starting address, and (2) size of the mshare'd region.

--
int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Example Code
============

Snippet of the code that a donor process would run looks like below:

-----------------

        addr = mmap((void *)TB(2), GB(512), PROT_READ | PROT_WRITE,
                        MAP_SHARED | MAP_ANONYMOUS, 0, 0);
        if (addr == MAP_FAILED)
                perror("ERROR: mmap failed");

        err = syscall(MSHARE_SYSCALL, "testregion", (void *)TB(2),
                        GB(512), O_CREAT|O_RDWR|O_EXCL, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        strncpy(addr, "Some random shared text",
                        sizeof("Some random shared text"));

-----------------

Snippet of code that a consumer process would execute looks like:

-----------------

        struct mshare_info minfo;

        fd = open("testregion", O_RDONLY);
        if (fd < 0) {
                perror("open failed");
                exit(1);
        }

        if ((count = read(fd, &minfo, sizeof(struct mshare_info)) > 0))
                printf("INFO: %ld bytes shared at addr 0x%lx \n",
                                minfo.size, minfo.start);
        else
                perror("read failed");

        close(fd);

        addr = (void *)minfo.start;
        err = syscall(MSHARE_SYSCALL, "testregion", addr, minfo.size,
                        O_RDWR, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        printf("Guest mmap at %px:\n", addr);
        printf("%s\n", addr);
        printf("\nDone\n");

        err = syscall(MSHARE_UNLINK_SYSCALL, "testregion");
        if (err < 0) {
                perror("mshare_unlink() failed");
                exit(1);
        }

-----------------

Patch series
============

This series of patches is an initial implementation of these two system calls. This code implements working basic functionality.

Prototype for the two syscalls is:

SYSCALL_DEFINE5(mshare, const char *, name, unsigned long, addr,
unsigned long, len, int, oflag, mode_t, mode)

SYSCALL_DEFINE1(mshare_unlink, const char *, name)

In order to facilitate page table sharing, this implemntation adds a new in-memory filesystem - msharefs which will be mounted at /sys/fs/mshare. When a new mshare'd region is created, a file with the name given by initial mshare() call is created under this filesystem. Permissions for this file are given by the "mode" parameter to mshare(). The only operation supported on this file is read. A read from this file returns a structure containing information about mshare'd region - (1) starting virtual address for the region, and (2) size of mshare'd region.

A donor process that wants to create an mshare'd region from a section of its mapped addresses calls mshare() with O_CREAT oflag. mshare() syscall then creates a new mm_struct which will host the page tables for the mshare'd region. vma->vm_private_data for the vmas covering address range for this region are updated to point to a structure containing pointer to this new mm_struct. Existing page tables are copied over to new mm struct.

A consumer process that wants to map mshare'd region opens /sys/fs/mshare/<filename> and reads the starting address and size of mshare'd region. It then calls mshare() with these values to map the entire region in its address space. Consumer process calls mshare_unlink() to terminate its access.

Since RFC
=========

This patch series includes better error handling and more robust locking besides improved implementation of mshare since the original RFC. It also incorporates feedback from original RFC. Alignment and size requirment are PGDIR_SIZE, same as RFC and this is open to change based upon further feedback. More review is needed for this patch series and is much appreciated.

Khalid Aziz (14):
mm: Add new system calls mshare, mshare_unlink
mm: Add msharefs filesystem
mm: Add read for msharefs
mm: implement mshare_unlink syscall
mm: Add locking to msharefs syscalls
mm/msharefs: Check for mounted filesystem
mm: Add vm flag for shared PTE
mm/mshare: Add basic page table sharing using mshare
mm: Do not free PTEs for mshare'd PTEs
mm/mshare: Check for mapped vma when mshare'ing existing mshare'd
range
mm/mshare: unmap vmas in mshare_unlink
mm/mshare: Add a proc file with mshare alignment/size information mm/mshare: Enforce mshare'd region permissions mm/mshare: Copy PTEs to host mm

 Documentation/filesystems/msharefs.rst |  19 +
 arch/x86/entry/syscalls/syscall_64.tbl |   2 +
 include/linux/mm.h                     |  11 +
 include/trace/events/mmflags.h         |   3 +-
 include/uapi/asm-generic/unistd.h      |   7 +-
 include/uapi/linux/magic.h             |   1 +
 include/uapi/linux/mman.h              |   5 +
 kernel/sysctl.c                        |   7 +
 mm/Makefile                            |   2 +-
 mm/internal.h                          |   7 +
 mm/memory.c                            | 105 ++++-
 mm/mshare.c                            | 587 +++++++++++++++++++++++++
 12 files changed, 750 insertions(+), 6 deletions(-)
 create mode 100644 Documentation/filesystems/msharefs.rst
 create mode 100644 mm/mshare.c

--
2.32.0

Quoted 156 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

A file under /sys/fs/mshare can be opened and read from. A read from this file returns two long values - (1) starting address, and (2) size of the mshare'd region.

--
int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Example Code
============

Snippet of the code that a donor process would run looks like below:

-----------------

        addr = mmap((void *)TB(2), GB(512), PROT_READ | PROT_WRITE,
                        MAP_SHARED | MAP_ANONYMOUS, 0, 0);
        if (addr == MAP_FAILED)
                perror("ERROR: mmap failed");

        err = syscall(MSHARE_SYSCALL, "testregion", (void *)TB(2),
                        GB(512), O_CREAT|O_RDWR|O_EXCL, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        strncpy(addr, "Some random shared text",
                        sizeof("Some random shared text"));

-----------------

Snippet of code that a consumer process would execute looks like:

-----------------

        struct mshare_info minfo;

        fd = open("testregion", O_RDONLY);
        if (fd < 0) {
                perror("open failed");
                exit(1);
        }

        if ((count = read(fd, &minfo, sizeof(struct mshare_info)) > 0))
                printf("INFO: %ld bytes shared at addr 0x%lx \n",
                                minfo.size, minfo.start);
        else
                perror("read failed");

        close(fd);

        addr = (void *)minfo.start;
        err = syscall(MSHARE_SYSCALL, "testregion", addr, minfo.size,
                        O_RDWR, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        printf("Guest mmap at %px:\n", addr);
        printf("%s\n", addr);
        printf("\nDone\n");

        err = syscall(MSHARE_UNLINK_SYSCALL, "testregion");
        if (err < 0) {
                perror("mshare_unlink() failed");
                exit(1);
        }

Quoted 1 line

-----------------

Does that mean those shared pages will get page_mapcount()=1 ?

A big pain for a memory limited system like a desktop/embedded system is that reverse mapping will take tons of cpu in memory reclamation path especially for those pages mapped by multiple processes. sometimes, 100% cpu utilization on LRU to scan and find out if a page is accessed by reading PTE young.

if we result in one PTE only by this patchset, it means we are getting significant
performance improvement in kernel LRU particularly when free memory approaches the watermarks.

But I don't see how a new system call like mshare(), can be used by those systems as they might need some more automatic PTEs sharing mechanism.

BTW, I suppose we are actually able to share PTEs as long as the address is 2MB aligned?

Quoted 160 lines · Barry Song

Quoted 156 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

A file under /sys/fs/mshare can be opened and read from. A read from this file returns two long values - (1) starting address, and (2) size of the mshare'd region.

--
int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Example Code
============

Snippet of the code that a donor process would run looks like below:

-----------------

        addr = mmap((void *)TB(2), GB(512), PROT_READ | PROT_WRITE,
                        MAP_SHARED | MAP_ANONYMOUS, 0, 0);
        if (addr == MAP_FAILED)
                perror("ERROR: mmap failed");

        err = syscall(MSHARE_SYSCALL, "testregion", (void *)TB(2),
                        GB(512), O_CREAT|O_RDWR|O_EXCL, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        strncpy(addr, "Some random shared text",
                        sizeof("Some random shared text"));

-----------------

Snippet of code that a consumer process would execute looks like:

-----------------

        struct mshare_info minfo;

        fd = open("testregion", O_RDONLY);
        if (fd < 0) {
                perror("open failed");
                exit(1);
        }

        if ((count = read(fd, &minfo, sizeof(struct mshare_info)) > 0))
                printf("INFO: %ld bytes shared at addr 0x%lx \n",
                                minfo.size, minfo.start);
        else
                perror("read failed");

        close(fd);

        addr = (void *)minfo.start;
        err = syscall(MSHARE_SYSCALL, "testregion", addr, minfo.size,
                        O_RDWR, 600);
        if (err < 0) {
                perror("mshare() syscall failed");
                exit(1);
        }

        printf("Guest mmap at %px:\n", addr);
        printf("%s\n", addr);
        printf("\nDone\n");

        err = syscall(MSHARE_UNLINK_SYSCALL, "testregion");
        if (err < 0) {
                perror("mshare_unlink() failed");
                exit(1);
        }

Quoted 1 line

-----------------

Quoted 1 line

Does that mean those shared pages will get page_mapcount()=1 ?

AFAIU, for mshare() that is the case.

Quoted 6 lines

A big pain for a memory limited system like a desktop/embedded system is that reverse mapping will take tons of cpu in memory reclamation path especially for those pages mapped by multiple processes. sometimes, 100% cpu utilization on LRU to scan and find out if a page is accessed by reading PTE young.

Regarding PTE-table sharing:

Even if we'd account each logical mapping (independent of page table sharing) in the page_mapcount(), we would benefit from page table sharing. Simply when we unmap the page from the shared page table, we'd have to adjust the mapcount accordingly. So unmapping from a single (shared) pagetable could directly result in the mapcount dropping to zero.

What I am trying to say is: how the mapcount is handled might be an implementation detail for PTE-sharing. Not sure how hugetlb handles that with its PMD-table sharing.

We'd have to clarify what the mapcount actually expresses. Having the mapcount express "is this page mapped by multiple processes or at multiple VMAs" might be helpful in some cases. Right now it mostly expresses exactly that.

Quoted 10 lines

if we result in one PTE only by this patchset, it means we are getting significant
performance improvement in kernel LRU particularly when free memory approaches the watermarks.

But I don't see how a new system call like mshare(), can be used by those systems as they might need some more automatic PTEs sharing mechanism.

IMHO, we should look into automatic PTE-table sharing of MAP_SHARED mappings, similar to what hugetlb provides for PMD table sharing, which leaves semantics unchanged for existing user space. Maybe there is a way to factor that out and reuse it for PTE-table sharing.

I can understand that there are use cases for explicit sharing with new (e.g., mprotect) semantics.

Quoted 3 lines

BTW, I suppose we are actually able to share PTEs as long as the address is 2MB aligned?

2MB is x86 specific, but yes. As long as we're aligned to PMDs and
* the VMA spans the complete PMD
* the VMA permissions match the page table
* no process-specific page table features are used (uffd, softdirty tracking)

... probably there are more corner cases. (e.g., referenced and dirty bit don't reflect what the mapping process did, which might or might not be relevant for current/future features)

--
Thanks,

David / dhildenb

Quoted 160 lines · Barry Song

Quoted 156 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

A file under /sys/fs/mshare can be opened and read from. A read from this file returns two long values - (1) starting address, and (2) size of the mshare'd region.

--
int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Example Code
============

Snippet of the code that a donor process would run looks like below:

-----------------

         addr = mmap((void *)TB(2), GB(512), PROT_READ | PROT_WRITE,
                         MAP_SHARED | MAP_ANONYMOUS, 0, 0);
         if (addr == MAP_FAILED)
                 perror("ERROR: mmap failed");

         err = syscall(MSHARE_SYSCALL, "testregion", (void *)TB(2),
                         GB(512), O_CREAT|O_RDWR|O_EXCL, 600);
         if (err < 0) {
                 perror("mshare() syscall failed");
                 exit(1);
         }

         strncpy(addr, "Some random shared text",
                         sizeof("Some random shared text"));

-----------------

Snippet of code that a consumer process would execute looks like:

-----------------

         struct mshare_info minfo;

         fd = open("testregion", O_RDONLY);
         if (fd < 0) {
                 perror("open failed");
                 exit(1);
         }

         if ((count = read(fd, &minfo, sizeof(struct mshare_info)) > 0))
                 printf("INFO: %ld bytes shared at addr 0x%lx \n",
                                 minfo.size, minfo.start);
         else
                 perror("read failed");

         close(fd);

         addr = (void *)minfo.start;
         err = syscall(MSHARE_SYSCALL, "testregion", addr, minfo.size,
                         O_RDWR, 600);
         if (err < 0) {
                 perror("mshare() syscall failed");
                 exit(1);
         }

         printf("Guest mmap at %px:\n", addr);
         printf("%s\n", addr);
         printf("\nDone\n");

         err = syscall(MSHARE_UNLINK_SYSCALL, "testregion");
         if (err < 0) {
                 perror("mshare_unlink() failed");
                 exit(1);
         }

Quoted 1 line

-----------------

Quoted 1 line

Does that mean those shared pages will get page_mapcount()=1 ?

AFAIU, for mshare() that is the case.

Quoted 6 lines

A big pain for a memory limited system like a desktop/embedded system is that reverse mapping will take tons of cpu in memory reclamation path especially for those pages mapped by multiple processes. sometimes, 100% cpu utilization on LRU to scan and find out if a page is accessed by reading PTE young.

Regarding PTE-table sharing:

Even if we'd account each logical mapping (independent of page table sharing) in the page_mapcount(), we would benefit from page table sharing. Simply when we unmap the page from the shared page table, we'd have to adjust the mapcount accordingly. So unmapping from a single (shared) pagetable could directly result in the mapcount dropping to zero.

What I am trying to say is: how the mapcount is handled might be an implementation detail for PTE-sharing. Not sure how hugetlb handles that with its PMD-table sharing.

We'd have to clarify what the mapcount actually expresses. Having the mapcount express "is this page mapped by multiple processes or at multiple VMAs" might be helpful in some cases. Right now it mostly expresses exactly that.

Quoted 10 lines

if we result in one PTE only by this patchset, it means we are getting significant
performance improvement in kernel LRU particularly when free memory approaches the watermarks.

But I don't see how a new system call like mshare(), can be used by those systems as they might need some more automatic PTEs sharing mechanism.

IMHO, we should look into automatic PTE-table sharing of MAP_SHARED mappings, similar to what hugetlb provides for PMD table sharing, which leaves semantics unchanged for existing user space. Maybe there is a way to factor that out and reuse it for PTE-table sharing.

I can understand that there are use cases for explicit sharing with new (e.g., mprotect) semantics.

Quoted 156 lines · Khalid Aziz

Page tables in kernel consume some of the memory and as long as number of mappings being maintained is small enough, this space consumed by page tables is not objectionable. When very few memory pages are shared between processes, the number of page table entries (PTEs) to maintain is mostly constrained by the number of pages of memory on the system. As the number of shared pages and the number of times pages are shared goes up, amount of memory consumed by page tables starts to become significant.

Some of the field deployments commonly see memory pages shared across 1000s of processes. On x86_64, each page requires a PTE that is only 8 bytes long which is very small compared to the 4K page size. When 2000 processes map the same page in their address space, each one of them requires 8 bytes for its PTE and together that adds up to 8K of memory just to hold the PTEs for one 4K page. On a database server with 300GB SGA, a system carsh was seen with out-of-memory condition when 1500+ clients tried to share this SGA even though the system had 512GB of memory. On this server, in the worst case scenario of all 1500 processes mapping every page from SGA would have required 878GB+ for just the PTEs. If these PTEs could be shared, amount of memory saved is very significant.

This patch series implements a mechanism in kernel to allow userspace processes to opt into sharing PTEs. It adds two new system calls - (1) mshare(), which can be used by a process to create a region (we will call it mshare'd region) which can be used by other processes to map same pages using shared PTEs, (2) mshare_unlink() which is used to detach from the mshare'd region. Once an mshare'd region is created, other process(es), assuming they have the right permissions, can make the mashare() system call to map the shared pages into their address space using the shared PTEs. When a process is done using this mshare'd region, it makes a mshare_unlink() system call to end its access. When the last process accessing mshare'd region calls mshare_unlink(), the mshare'd region is torn down and memory used by it is freed.

API
===

The mshare API consists of two system calls - mshare() and mshare_unlink()

--
int mshare(char *name, void *addr, size_t length, int oflags, mode_t mode)

mshare() creates and opens a new, or opens an existing mshare'd region that will be shared at PTE level. "name" refers to shared object name that exists under /sys/fs/mshare. "addr" is the starting address of this shared memory area and length is the size of this area. oflags can be one of:

- O_RDONLY opens shared memory area for read only access by everyone
- O_RDWR opens shared memory area for read and write access
- O_CREAT creates the named shared memory area if it does not exist
- O_EXCL If O_CREAT was also specified, and a shared memory area exists with that name, return an error.

mode represents the creation mode for the shared object under /sys/fs/mshare.

mshare() returns an error code if it fails, otherwise it returns 0.

PTEs are shared at pgdir level and hence it imposes following requirements on the address and size given to the mshare():

- Starting address must be aligned to pgdir size (512GB on x86_64). This alignment value can be looked up in /proc/sys/vm//mshare_size
- Size must be a multiple of pgdir size
- Any mappings created in this address range at any time become shared automatically
- Shared address range can have unmapped addresses in it. Any access to unmapped address will result in SIGBUS

Mappings within this address range behave as if they were shared between threads, so a write to a MAP_PRIVATE mapping will create a page which is shared between all the sharers. The first process that declares an address range mshare'd can continue to map objects in the shared area. All other processes that want mshare'd access to this memory area can do so by calling mshare(). After this call, the address range given by mshare becomes a shared range in its address space. Anonymous mappings will be shared and not COWed.

A file under /sys/fs/mshare can be opened and read from. A read from this file returns two long values - (1) starting address, and (2) size of the mshare'd region.

--
int mshare_unlink(char *name)

A shared address range created by mshare() can be destroyed using mshare_unlink() which removes the shared named object. Once all processes have unmapped the shared object, the shared address range references are de-allocated and destroyed.

mshare_unlink() returns 0 on success or -1 on error.

Example Code
============

Snippet of the code that a donor process would run looks like below:

-----------------

         addr = mmap((void *)TB(2), GB(512), PROT_READ | PROT_WRITE,
                         MAP_SHARED | MAP_ANONYMOUS, 0, 0);
         if (addr == MAP_FAILED)
                 perror("ERROR: mmap failed");

         err = syscall(MSHARE_SYSCALL, "testregion", (void *)TB(2),
                         GB(512), O_CREAT|O_RDWR|O_EXCL, 600);
         if (err < 0) {
                 perror("mshare() syscall failed");
                 exit(1);
         }

         strncpy(addr, "Some random shared text",
                         sizeof("Some random shared text"));

-----------------

Snippet of code that a consumer process would execute looks like:

-----------------

         struct mshare_info minfo;

         fd = open("testregion", O_RDONLY);
         if (fd < 0) {
                 perror("open failed");
                 exit(1);
         }

         if ((count = read(fd, &minfo, sizeof(struct mshare_info)) > 0))
                 printf("INFO: %ld bytes shared at addr 0x%lx \n",
                                 minfo.size, minfo.start);
         else
                 perror("read failed");

         close(fd);

         addr = (void *)minfo.start;
         err = syscall(MSHARE_SYSCALL, "testregion", addr, minfo.size,
                         O_RDWR, 600);
         if (err < 0) {
                 perror("mshare() syscall failed");
                 exit(1);
         }

         printf("Guest mmap at %px:\n", addr);
         printf("%s\n", addr);
         printf("\nDone\n");

         err = syscall(MSHARE_UNLINK_SYSCALL, "testregion");
         if (err < 0) {
                 perror("mshare_unlink() failed");
                 exit(1);
         }

Quoted 1 line

-----------------

Does that mean those shared pages will get page_mapcount()=1 ?

A big pain for a memory limited system like a desktop/embedded system is that reverse mapping will take tons of cpu in memory reclamation path especially for those pages mapped by multiple processes. sometimes, 100% cpu utilization on LRU to scan and find out if a page is accessed by reading PTE young.

if we result in one PTE only by this patchset, it means we are getting significant
performance improvement in kernel LRU particularly when free memory approaches the watermarks.

But I don't see how a new system call like mshare(), can be used by those systems as they might need some more automatic PTEs sharing mechanism.

BTW, I suppose we are actually able to share PTEs as long as the address is 2MB aligned?