HMM (Heterogeneous Memory Management) v8

From design point of view not much changed since last patchset (2).

From: JA(C)rA'me Glisse <[email protected]>

The event information will be useful for new user of mmu_notifier API. The event argument differentiate between a vma disappearing, a page being write protected or simply a page being unmaped. This allow new user to take different path for different event for instance on unmap the resource used to track a vma are still valid and should stay around. While if the event is saying that a vma is being destroy it means that any resources used to track this vma can be free.

Changed since v1:
- renamed action into event (updated commit message too).
- simplified the event names and clarified their usage also documenting what exceptation the listener can have in respect to each event.

Changed since v2:
- Avoid crazy name.
- Do not move code that do not need to move.

Changed since v3:
- Separate hugue page split from mlock/munlock and softdirty.

Changed since v4:
- Rebase (no other changes).

Changed since v5:
- Typo fix.
- Changed zap_page_range from MMU_MUNMAP to MMU_MIGRATE to reflect the fact that the address range is still valid just the page backing it are no longer.

Changed since v6:
- try_to_unmap_one() only invalidate when doing migration.
- Differentiate fork from other case.

Signed-off-by JA(C)rA'me Glisse <[email protected]>

Reviewed-by Rik van Riel <[email protected]>

---
 drivers/gpu/drm/i915/i915_gem_userptr.c |   3 +-
 drivers/gpu/drm/radeon/radeon_mn.c      |   3 +-
 drivers/infiniband/core/umem_odp.c      |   9 ++-
 drivers/iommu/amd_iommu_v2.c            |   3 +-
 drivers/misc/sgi-gru/grutlbpurge.c      |   9 ++-
 drivers/xen/gntdev.c                    |   9 ++-
 fs/proc/task_mmu.c                      |   6 +-
 include/linux/mmu_notifier.h            | 135 ++++++++++++++++++++++++++------
 kernel/events/uprobes.c                 |  10 ++-
 mm/huge_memory.c                        |  39 ++++++---
 mm/hugetlb.c                            |  23 +++---
 mm/ksm.c                                |  18 +++--
 mm/madvise.c                            |   4 +-
 mm/memory.c                             |  27 ++++---
 mm/migrate.c                            |   9 ++-
 mm/mmu_notifier.c                       |  28 ++++---
 mm/mprotect.c                           |   6 +-
 mm/mremap.c                             |   6 +-
 mm/rmap.c                               |   4 +-
 virt/kvm/kvm_main.c                     |  12 ++-
 20 files changed, 261 insertions(+), 102 deletions(-)

drivers/gpu/drm/i915/i915_gem_userptr.c +2 -1

index 4039ede..452e9b1 100644
@@ -132,7 +132,8 @@ restart:
 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 						       struct mm_struct *mm,
 						       unsigned long start,
-						       unsigned long end)
+						       unsigned long end,
+						       enum mmu_event event)
 {
 	struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn);
 	struct interval_tree_node *it = NULL;

drivers/gpu/drm/radeon/radeon_mn.c +2 -1

index eef006c..3a9615b 100644
@@ -121,7 +121,8 @@ static void radeon_mn_release(struct mmu_notifier *mn,
 static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long start,
-					     unsigned long end)
+					     unsigned long end,
+					     enum mmu_event event)
 {
 	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
 	struct interval_tree_node *it;

drivers/infiniband/core/umem_odp.c +6 -3

index 40becdb..6ed69fa 100644
@@ -165,7 +165,8 @@ static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
 
 static void ib_umem_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
-					     unsigned long address)
+					     unsigned long address,
+					     enum mmu_event event)
 {
 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
 

@@ -192,7 +193,8 @@ static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start,
 static void ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
 						    struct mm_struct *mm,
 						    unsigned long start,
-						    unsigned long end)
+						    unsigned long end,
+						    enum mmu_event event)
 {
 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
 

@@ -217,7 +219,8 @@ static int invalidate_range_end_trampoline(struct ib_umem *item, u64 start,
 static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
 						  struct mm_struct *mm,
 						  unsigned long start,
-						  unsigned long end)
+						  unsigned long end,
+						  enum mmu_event event)
 {
 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
 

drivers/iommu/amd_iommu_v2.c +2 -1

index 3465faf..4aa4de6 100644
@@ -384,7 +384,8 @@ static int mn_clear_flush_young(struct mmu_notifier *mn,
 
 static void mn_invalidate_page(struct mmu_notifier *mn,
 			       struct mm_struct *mm,
-			       unsigned long address)
+			       unsigned long address,
+			       enum mmu_event event)
 {
 	__mn_flush_page(mn, address);
 }

drivers/misc/sgi-gru/grutlbpurge.c +6 -3

index 2129274..e67fed1 100644
@@ -221,7 +221,8 @@ void gru_flush_all_tlb(struct gru_state *gru)
  */
 static void gru_invalidate_range_start(struct mmu_notifier *mn,
 				       struct mm_struct *mm,
-				       unsigned long start, unsigned long end)
+				       unsigned long start, unsigned long end,
+				       enum mmu_event event)
 {
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
 						 ms_notifier);

@@ -235,7 +236,8 @@ static void gru_invalidate_range_start(struct mmu_notifier *mn,
 
 static void gru_invalidate_range_end(struct mmu_notifier *mn,
 				     struct mm_struct *mm, unsigned long start,
-				     unsigned long end)
+				     unsigned long end,
+				     enum mmu_event event)
 {
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
 						 ms_notifier);

@@ -248,7 +250,8 @@ static void gru_invalidate_range_end(struct mmu_notifier *mn,
 }
 
 static void gru_invalidate_page(struct mmu_notifier *mn, struct mm_struct *mm,
-				unsigned long address)
+				unsigned long address,
+				enum mmu_event event)
 {
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
 						 ms_notifier);

drivers/xen/gntdev.c +6 -3

index 8927485..46bc610 100644
@@ -467,7 +467,9 @@ static void unmap_if_in_range(struct grant_map *map,
 
 static void mn_invl_range_start(struct mmu_notifier *mn,
 				struct mm_struct *mm,
-				unsigned long start, unsigned long end)
+				unsigned long start,
+				unsigned long end,
+				enum mmu_event event)
 {
 	struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
 	struct grant_map *map;

@@ -484,9 +486,10 @@ static void mn_invl_range_start(struct mmu_notifier *mn,
 
 static void mn_invl_page(struct mmu_notifier *mn,
 			 struct mm_struct *mm,
-			 unsigned long address)
+			 unsigned long address,
+			 enum mmu_event event)
 {
-	mn_invl_range_start(mn, mm, address, address + PAGE_SIZE);
+	mn_invl_range_start(mn, mm, address, address + PAGE_SIZE, event);
 }
 
 static void mn_release(struct mmu_notifier *mn,

fs/proc/task_mmu.c +4 -2

index 6dee68d..58e2390 100644
@@ -934,11 +934,13 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
 				downgrade_write(&mm->mmap_sem);
 				break;
 			}
-			mmu_notifier_invalidate_range_start(mm, 0, -1);
+			mmu_notifier_invalidate_range_start(mm, 0,
+							    -1, MMU_ISDIRTY);
 		}
 		walk_page_range(0, ~0UL, &clear_refs_walk);
 		if (type == CLEAR_REFS_SOFT_DIRTY)
-			mmu_notifier_invalidate_range_end(mm, 0, -1);
+			mmu_notifier_invalidate_range_end(mm, 0,
+							  -1, MMU_ISDIRTY);
 		flush_tlb_mm(mm);
 		up_read(&mm->mmap_sem);
 out_mm:

include/linux/mmu_notifier.h +0 -0

index 61cd67f..8b11b1b 100644

include/linux/mmu_notifier.h +36 -0

+ *
+ *   - MMU_ISDIRTY need to update the dirty bit of the page table so proper
+ *     dirty accounting can happen.
+ *
+ *   - MMU_MIGRATE: memory is migrating from one page to another, thus all write
+ *     access must stop after invalidate_range_start callback returns.
+ *     Furthermore, no read access should be allowed either, as a new page can
+ *     be remapped with write access before the invalidate_range_end callback
+ *     happens and thus any read access to old page might read stale data. There
+ *     are several sources for this event, including:
+ *
+ *         - A page moving to swap (various reasons, including page reclaim),
+ *         - An mremap syscall,
+ *         - migration for NUMA reasons,
+ *         - balancing the memory pool,
+ *         - write fault on COW page,
+ *         - and more that are not listed here.
+ *
+ *   - MMU_MPROT: memory access protection is changing. Refer to the vma to get
+ *     the new access protection. All memory access are still valid until the
+ *     invalidate_range_end callback.
+ *
+ *   - MMU_MUNLOCK: unlock memory. Content of page table stays the same but
+ *     page are unlocked.
+ *
+ *   - MMU_MUNMAP: the range is being unmapped (outcome of a munmap syscall or
+ *     process destruction). However, access is still allowed, up until the
+ *     invalidate_range_free_pages callback. This also implies that secondary
+ *     page table can be trimmed, because the address range is no longer valid.
+ *
+ *   - MMU_WRITE_BACK: memory is being written back to disk, all write accesses
+ *     must stop after invalidate_range_start callback returns. Read access are
+ *     still allowed.
+ *
+ *   - MMU_WRITE_PROTECT: memory is being write protected (ie should be mapped
+ *     read only no matter what the vma memory protection allows). All write

include/linux/mmu_notifier.h +51 -22

+};
+
 #ifdef CONFIG_MMU_NOTIFIER
 
 /*

@@ -82,7 +146,8 @@ struct mmu_notifier_ops {
 	void (*change_pte)(struct mmu_notifier *mn,
 			   struct mm_struct *mm,
 			   unsigned long address,
-			   pte_t pte);
+			   pte_t pte,
+			   enum mmu_event event);
 
 	/*
 	 * Before this is invoked any secondary MMU is still ok to

@@ -93,7 +158,8 @@ struct mmu_notifier_ops {
 	 */
 	void (*invalidate_page)(struct mmu_notifier *mn,
 				struct mm_struct *mm,
-				unsigned long address);
+				unsigned long address,
+				enum mmu_event event);
 
 	/*
 	 * invalidate_range_start() and invalidate_range_end() must be

@@ -140,10 +206,14 @@ struct mmu_notifier_ops {
 	 */
 	void (*invalidate_range_start)(struct mmu_notifier *mn,
 				       struct mm_struct *mm,
-				       unsigned long start, unsigned long end);
+				       unsigned long start,
+				       unsigned long end,
+				       enum mmu_event event);
 	void (*invalidate_range_end)(struct mmu_notifier *mn,
 				     struct mm_struct *mm,
-				     unsigned long start, unsigned long end);
+				     unsigned long start,
+				     unsigned long end,
+				     enum mmu_event event);
 
 	/*
 	 * invalidate_range() is either called between

@@ -206,13 +276,20 @@ extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
 extern int __mmu_notifier_test_young(struct mm_struct *mm,
 				     unsigned long address);
 extern void __mmu_notifier_change_pte(struct mm_struct *mm,
-				      unsigned long address, pte_t pte);
+				      unsigned long address,
+				      pte_t pte,
+				      enum mmu_event event);
 extern void __mmu_notifier_invalidate_page(struct mm_struct *mm,
-					  unsigned long address);
+					  unsigned long address,
+					  enum mmu_event event);
 extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
-				  unsigned long start, unsigned long end);
+						  unsigned long start,
+						  unsigned long end,
+						  enum mmu_event event);
 extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-				  unsigned long start, unsigned long end);
+						unsigned long start,
+						unsigned long end,
+						enum mmu_event event);
 extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
 				  unsigned long start, unsigned long end);
 

@@ -240,31 +317,38 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm,
 }
 
 static inline void mmu_notifier_change_pte(struct mm_struct *mm,
-					   unsigned long address, pte_t pte)
+					   unsigned long address,
+					   pte_t pte,
+					   enum mmu_event event)
 {
 	if (mm_has_notifiers(mm))
-		__mmu_notifier_change_pte(mm, address, pte);
+		__mmu_notifier_change_pte(mm, address, pte, event);
 }
 
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
-					  unsigned long address)
+						unsigned long address,
+						enum mmu_event event)
 {
 	if (mm_has_notifiers(mm))
-		__mmu_notifier_invalidate_page(mm, address);
+		__mmu_notifier_invalidate_page(mm, address, event);
 }
 
 static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+						       unsigned long start,
+						       unsigned long end,
+						       enum mmu_event event)
 {
 	if (mm_has_notifiers(mm))
-		__mmu_notifier_invalidate_range_start(mm, start, end);
+		__mmu_notifier_invalidate_range_start(mm, start, end, event);
 }
 
 static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+						     unsigned long start,
+						     unsigned long end,
+						     enum mmu_event event)
 {
 	if (mm_has_notifiers(mm))
-		__mmu_notifier_invalidate_range_end(mm, start, end);
+		__mmu_notifier_invalidate_range_end(mm, start, end, event);
 }
 
 static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,

@@ -359,13 +443,13 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
  * old page would remain mapped readonly in the secondary MMUs after the new
  * page is already writable by some CPU through the primary MMU.
  */
-#define set_pte_at_notify(__mm, __address, __ptep, __pte)		\
+#define set_pte_at_notify(__mm, __address, __ptep, __pte, __event)	\
 ({									\
 	struct mm_struct *___mm = __mm;					\
 	unsigned long ___address = __address;				\
 	pte_t ___pte = __pte;						\
 									\
-	mmu_notifier_change_pte(___mm, ___address, ___pte);		\
+	mmu_notifier_change_pte(___mm, ___address, ___pte, __event);	\
 	set_pte_at(___mm, ___address, __ptep, ___pte);			\
 })
 

@@ -393,22 +477,29 @@ static inline int mmu_notifier_test_young(struct mm_struct *mm,
 }
 
 static inline void mmu_notifier_change_pte(struct mm_struct *mm,
-					   unsigned long address, pte_t pte)
+					   unsigned long address,
+					   pte_t pte,
+					   enum mmu_event event)
 {
 }
 
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
-					  unsigned long address)
+						unsigned long address,
+						enum mmu_event event)
 {
 }
 
 static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+						       unsigned long start,
+						       unsigned long end,
+						       enum mmu_event event)
 {
 }
 
 static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+						     unsigned long start,
+						     unsigned long end,
+						     enum mmu_event event)
 {
 }
 

kernel/events/uprobes.c +7 -3

index cb346f2..802828a 100644
@@ -176,7 +176,8 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 	/* For try_to_free_swap() and munlock_vma_page() below */
 	lock_page(page);
 
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 	err = -EAGAIN;
 	ptep = page_check_address(page, mm, addr, &ptl, 0);
 	if (!ptep)

@@ -194,7 +195,9 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 
 	flush_cache_page(vma, addr, pte_pfn(*ptep));
 	ptep_clear_flush_notify(vma, addr, ptep);
-	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
+	set_pte_at_notify(mm, addr, ptep,
+			  mk_pte(kpage, vma->vm_page_prot),
+			  MMU_MIGRATE);
 
 	page_remove_rmap(page);
 	if (!page_mapped(page))

@@ -208,7 +211,8 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 	err = 0;
  unlock:
 	mem_cgroup_cancel_charge(kpage, memcg);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 	unlock_page(page);
 	return err;
 }

mm/huge_memory.c +26 -13

index cb8904c..41c342c 100644
@@ -1024,7 +1024,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
 
 	mmun_start = haddr;
 	mmun_end   = haddr + HPAGE_PMD_SIZE;
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
+					    MMU_MIGRATE);
 
 	ptl = pmd_lock(mm, pmd);
 	if (unlikely(!pmd_same(*pmd, orig_pmd)))

@@ -1058,7 +1059,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
 	page_remove_rmap(page);
 	spin_unlock(ptl);
 
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 
 	ret |= VM_FAULT_WRITE;
 	put_page(page);

@@ -1068,7 +1070,8 @@ out:
 
 out_free_pages:
 	spin_unlock(ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 	for (i = 0; i < HPAGE_PMD_NR; i++) {
 		memcg = (void *)page_private(pages[i]);
 		set_page_private(pages[i], 0);

@@ -1160,7 +1163,8 @@ alloc:
 
 	mmun_start = haddr;
 	mmun_end   = haddr + HPAGE_PMD_SIZE;
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
+					    MMU_MIGRATE);
 
 	spin_lock(ptl);
 	if (page)

@@ -1192,7 +1196,8 @@ alloc:
 	}
 	spin_unlock(ptl);
 out_mn:
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 out:
 	return ret;
 out_unlock:

@@ -1646,7 +1651,8 @@ static int __split_huge_page_splitting(struct page *page,
 	const unsigned long mmun_start = address;
 	const unsigned long mmun_end   = address + HPAGE_PMD_SIZE;
 
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_HSPLIT);
 	pmd = page_check_address_pmd(page, mm, address,
 			PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, &ptl);
 	if (pmd) {

@@ -1662,7 +1668,8 @@ static int __split_huge_page_splitting(struct page *page,
 		ret = 1;
 		spin_unlock(ptl);
 	}
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_HSPLIT);
 
 	return ret;
 }

@@ -2526,7 +2533,8 @@ static void collapse_huge_page(struct mm_struct *mm,
 
 	mmun_start = address;
 	mmun_end   = address + HPAGE_PMD_SIZE;
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
 	/*
 	 * After this gup_fast can't run anymore. This also removes

@@ -2536,7 +2544,8 @@ static void collapse_huge_page(struct mm_struct *mm,
 	 */
 	_pmd = pmdp_collapse_flush(vma, address, pmd);
 	spin_unlock(pmd_ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 
 	spin_lock(pte_ptl);
 	isolated = __collapse_huge_page_isolate(vma, address, pte);

@@ -2933,24 +2942,28 @@ void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
 	mmun_start = haddr;
 	mmun_end   = haddr + HPAGE_PMD_SIZE;
 again:
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 	ptl = pmd_lock(mm, pmd);
 	if (unlikely(!pmd_trans_huge(*pmd))) {
 		spin_unlock(ptl);
-		mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_end(mm, mmun_start,
+						  mmun_end, MMU_MIGRATE);
 		return;
 	}
 	if (is_huge_zero_pmd(*pmd)) {
 		__split_huge_zero_page_pmd(vma, haddr, pmd);
 		spin_unlock(ptl);
-		mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_end(mm, mmun_start,
+						  mmun_end, MMU_MIGRATE);
 		return;
 	}
 	page = pmd_page(*pmd);
 	VM_BUG_ON_PAGE(!page_count(page), page);
 	get_page(page);
 	spin_unlock(ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 
 	split_huge_page(page);
 

mm/hugetlb.c +14 -9

index 54f129d..19da310 100644
@@ -2670,7 +2670,8 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 	mmun_start = vma->vm_start;
 	mmun_end = vma->vm_end;
 	if (cow)
-		mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_start(src, mmun_start,
+						    mmun_end, MMU_MIGRATE);
 
 	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
 		spinlock_t *src_ptl, *dst_ptl;

@@ -2724,7 +2725,8 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 	}
 
 	if (cow)
-		mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_end(src, mmun_start,
+						  mmun_end, MMU_MIGRATE);
 
 	return ret;
 }

@@ -2750,7 +2752,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	BUG_ON(end & ~huge_page_mask(h));
 
 	tlb_start_vma(tlb, vma);
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 	address = start;
 again:
 	for (; address < end; address += sz) {

@@ -2824,7 +2827,8 @@ unlock:
 		if (address < end && !ref_page)
 			goto again;
 	}
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 	tlb_end_vma(tlb, vma);
 }
 

@@ -3003,8 +3007,8 @@ retry_avoidcopy:
 
 	mmun_start = address & huge_page_mask(h);
 	mmun_end = mmun_start + huge_page_size(h);
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
-
+	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
+					    MMU_MIGRATE);
 	/*
 	 * Retake the page table lock to check for racing updates
 	 * before the page tables are altered

@@ -3025,7 +3029,8 @@ retry_avoidcopy:
 		new_page = old_page;
 	}
 	spin_unlock(ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
+					  MMU_MIGRATE);
 out_release_all:
 	page_cache_release(new_page);
 out_release_old:

@@ -3493,7 +3498,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 	BUG_ON(address >= end);
 	flush_cache_range(vma, address, end);
 
-	mmu_notifier_invalidate_range_start(mm, start, end);
+	mmu_notifier_invalidate_range_start(mm, start, end, MMU_MPROT);
 	i_mmap_lock_write(vma->vm_file->f_mapping);
 	for (; address < end; address += huge_page_size(h)) {
 		spinlock_t *ptl;

@@ -3543,7 +3548,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 	flush_tlb_range(vma, start, end);
 	mmu_notifier_invalidate_range(mm, start, end);
 	i_mmap_unlock_write(vma->vm_file->f_mapping);
-	mmu_notifier_invalidate_range_end(mm, start, end);
+	mmu_notifier_invalidate_range_end(mm, start, end, MMU_MPROT);
 
 	return pages << h->order;
 }

mm/ksm.c +12 -6

index bc7be0e..76f167c 100644
@@ -872,7 +872,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
 
 	mmun_start = addr;
 	mmun_end   = addr + PAGE_SIZE;
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
+					    MMU_WRITE_PROTECT);
 
 	ptep = page_check_address(page, mm, addr, &ptl, 0);
 	if (!ptep)

@@ -904,7 +905,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
 		if (pte_dirty(entry))
 			set_page_dirty(page);
 		entry = pte_mkclean(pte_wrprotect(entry));
-		set_pte_at_notify(mm, addr, ptep, entry);
+		set_pte_at_notify(mm, addr, ptep, entry, MMU_WRITE_PROTECT);
 	}
 	*orig_pte = *ptep;
 	err = 0;

@@ -912,7 +913,8 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
 out_unlock:
 	pte_unmap_unlock(ptep, ptl);
 out_mn:
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
+					  MMU_WRITE_PROTECT);
 out:
 	return err;
 }

@@ -948,7 +950,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
 
 	mmun_start = addr;
 	mmun_end   = addr + PAGE_SIZE;
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end,
+					    MMU_MIGRATE);
 
 	ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
 	if (!pte_same(*ptep, orig_pte)) {

@@ -961,7 +964,9 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
 
 	flush_cache_page(vma, addr, pte_pfn(*ptep));
 	ptep_clear_flush_notify(vma, addr, ptep);
-	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
+	set_pte_at_notify(mm, addr, ptep,
+			  mk_pte(kpage, vma->vm_page_prot),
+			  MMU_MIGRATE);
 
 	page_remove_rmap(page);
 	if (!page_mapped(page))

@@ -971,7 +976,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
 	pte_unmap_unlock(ptep, ptl);
 	err = 0;
 out_mn:
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end,
+					  MMU_MIGRATE);
 out:
 	return err;
 }

mm/madvise.c +2 -2

index 22e8f0c..b90ba3d 100644
@@ -405,9 +405,9 @@ static int madvise_free_single_vma(struct vm_area_struct *vma,
 	tlb_gather_mmu(&tlb, mm, start, end);
 	update_hiwater_rss(mm);
 
-	mmu_notifier_invalidate_range_start(mm, start, end);
+	mmu_notifier_invalidate_range_start(mm, start, end, MMU_MUNMAP);
 	madvise_free_page_range(&tlb, vma, start, end);
-	mmu_notifier_invalidate_range_end(mm, start, end);
+	mmu_notifier_invalidate_range_end(mm, start, end, MMU_MUNMAP);
 	tlb_finish_mmu(&tlb, start, end);
 
 	return 0;

mm/memory.c +16 -11

index d1fa0c1..9300fad 100644
@@ -1048,7 +1048,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	mmun_end   = end;
 	if (is_cow)
 		mmu_notifier_invalidate_range_start(src_mm, mmun_start,
-						    mmun_end);
+						    mmun_end, MMU_FORK);
 
 	ret = 0;
 	dst_pgd = pgd_offset(dst_mm, addr);

@@ -1065,7 +1065,8 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
 
 	if (is_cow)
-		mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_end(src_mm, mmun_start,
+						  mmun_end, MMU_FORK);
 	return ret;
 }
 

@@ -1335,10 +1336,12 @@ void unmap_vmas(struct mmu_gather *tlb,
 {
 	struct mm_struct *mm = vma->vm_mm;
 
-	mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
+	mmu_notifier_invalidate_range_start(mm, start_addr,
+					    end_addr, MMU_MUNMAP);
 	for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
 		unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
-	mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
+	mmu_notifier_invalidate_range_end(mm, start_addr,
+					  end_addr, MMU_MUNMAP);
 }
 
 /**

@@ -1360,10 +1363,10 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
 	lru_add_drain();
 	tlb_gather_mmu(&tlb, mm, start, end);
 	update_hiwater_rss(mm);
-	mmu_notifier_invalidate_range_start(mm, start, end);
+	mmu_notifier_invalidate_range_start(mm, start, end, MMU_MIGRATE);
 	for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
 		unmap_single_vma(&tlb, vma, start, end, details);
-	mmu_notifier_invalidate_range_end(mm, start, end);
+	mmu_notifier_invalidate_range_end(mm, start, end, MMU_MIGRATE);
 	tlb_finish_mmu(&tlb, start, end);
 }
 

@@ -1386,9 +1389,9 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
 	lru_add_drain();
 	tlb_gather_mmu(&tlb, mm, address, end);
 	update_hiwater_rss(mm);
-	mmu_notifier_invalidate_range_start(mm, address, end);
+	mmu_notifier_invalidate_range_start(mm, address, end, MMU_MUNMAP);
 	unmap_single_vma(&tlb, vma, address, end, details);
-	mmu_notifier_invalidate_range_end(mm, address, end);
+	mmu_notifier_invalidate_range_end(mm, address, end, MMU_MUNMAP);
 	tlb_finish_mmu(&tlb, address, end);
 }
 

@@ -2086,7 +2089,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
 	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg))
 		goto oom_free_new;
 
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 
 	/*
 	 * Re-check the pte - we dropped the lock

@@ -2119,7 +2123,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
 		 * mmu page tables (such as kvm shadow page tables), we want the
 		 * new page to be mapped directly into the secondary page table.
 		 */
-		set_pte_at_notify(mm, address, page_table, entry);
+		set_pte_at_notify(mm, address, page_table, entry, MMU_MIGRATE);
 		update_mmu_cache(vma, address, page_table);
 		if (old_page) {
 			/*

@@ -2158,7 +2162,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
 		page_cache_release(new_page);
 
 	pte_unmap_unlock(page_table, ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 	if (old_page) {
 		/*
 		 * Don't let another task, with possibly unlocked vma,

mm/migrate.c +6 -3

index 236ee25..ad9a55a 100644
@@ -1759,12 +1759,14 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 	WARN_ON(PageLRU(new_page));
 
 	/* Recheck the target PMD */
-	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 	ptl = pmd_lock(mm, pmd);
 	if (unlikely(!pmd_same(*pmd, entry) || page_count(page) != 2)) {
 fail_putback:
 		spin_unlock(ptl);
-		mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+		mmu_notifier_invalidate_range_end(mm, mmun_start,
+						  mmun_end, MMU_MIGRATE);
 
 		/* Reverse changes made by migrate_page_copy() */
 		if (TestClearPageActive(new_page))

@@ -1818,7 +1820,8 @@ fail_putback:
 	page_remove_rmap(page);
 
 	spin_unlock(ptl);
-	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 
 	/* Take an "isolate" reference and put new page on the LRU. */
 	get_page(new_page);

mm/mmu_notifier.c +19 -9

index 3b9b3d0..e51ea02 100644
@@ -142,8 +142,10 @@ int __mmu_notifier_test_young(struct mm_struct *mm,
 	return young;
 }
 
-void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
-			       pte_t pte)
+void __mmu_notifier_change_pte(struct mm_struct *mm,
+			       unsigned long address,
+			       pte_t pte,
+			       enum mmu_event event)
 {
 	struct mmu_notifier *mn;
 	int id;

@@ -151,13 +153,14 @@ void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
 	id = srcu_read_lock(&srcu);
 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
 		if (mn->ops->change_pte)
-			mn->ops->change_pte(mn, mm, address, pte);
+			mn->ops->change_pte(mn, mm, address, pte, event);
 	}
 	srcu_read_unlock(&srcu, id);
 }
 
 void __mmu_notifier_invalidate_page(struct mm_struct *mm,
-					  unsigned long address)
+				    unsigned long address,
+				    enum mmu_event event)
 {
 	struct mmu_notifier *mn;
 	int id;

@@ -165,13 +168,16 @@ void __mmu_notifier_invalidate_page(struct mm_struct *mm,
 	id = srcu_read_lock(&srcu);
 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
 		if (mn->ops->invalidate_page)
-			mn->ops->invalidate_page(mn, mm, address);
+			mn->ops->invalidate_page(mn, mm, address, event);
 	}
 	srcu_read_unlock(&srcu, id);
 }
 
 void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+					   unsigned long start,
+					   unsigned long end,
+					   enum mmu_event event)
+
 {
 	struct mmu_notifier *mn;
 	int id;

@@ -179,14 +185,17 @@ void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
 	id = srcu_read_lock(&srcu);
 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
 		if (mn->ops->invalidate_range_start)
-			mn->ops->invalidate_range_start(mn, mm, start, end);
+			mn->ops->invalidate_range_start(mn, mm, start,
+							end, event);
 	}
 	srcu_read_unlock(&srcu, id);
 }
 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
 
 void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-				  unsigned long start, unsigned long end)
+					 unsigned long start,
+					 unsigned long end,
+					 enum mmu_event event)
 {
 	struct mmu_notifier *mn;
 	int id;

@@ -204,7 +213,8 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 		if (mn->ops->invalidate_range)
 			mn->ops->invalidate_range(mn, mm, start, end);
 		if (mn->ops->invalidate_range_end)
-			mn->ops->invalidate_range_end(mn, mm, start, end);
+			mn->ops->invalidate_range_end(mn, mm, start,
+						      end, event);
 	}
 	srcu_read_unlock(&srcu, id);
 }

mm/mprotect.c +4 -2

index e7d6f11..a57e8af 100644
@@ -155,7 +155,8 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
 		/* invoke the mmu notifier if the pmd is populated */
 		if (!mni_start) {
 			mni_start = addr;
-			mmu_notifier_invalidate_range_start(mm, mni_start, end);
+			mmu_notifier_invalidate_range_start(mm, mni_start,
+							    end, MMU_MPROT);
 		}
 
 		if (pmd_trans_huge(*pmd)) {

@@ -183,7 +184,8 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
 	} while (pmd++, addr = next, addr != end);
 
 	if (mni_start)
-		mmu_notifier_invalidate_range_end(mm, mni_start, end);
+		mmu_notifier_invalidate_range_end(mm, mni_start, end,
+						  MMU_MPROT);
 
 	if (nr_huge_updates)
 		count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);

mm/mremap.c +4 -2

index a7c93ec..72051cf 100644
@@ -176,7 +176,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
 
 	mmun_start = old_addr;
 	mmun_end   = old_end;
-	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start,
+					    mmun_end, MMU_MIGRATE);
 
 	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
 		cond_resched();

@@ -228,7 +229,8 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
 	if (likely(need_flush))
 		flush_tlb_range(vma, old_end-len, old_addr);
 
-	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
+	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start,
+					  mmun_end, MMU_MIGRATE);
 
 	return len + old_addr - old_end;	/* how much done */
 }

mm/rmap.c +2 -2

index 9c04594..74c51e0 100644
@@ -915,7 +915,7 @@ static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
 	pte_unmap_unlock(pte, ptl);
 
 	if (ret) {
-		mmu_notifier_invalidate_page(mm, address);
+		mmu_notifier_invalidate_page(mm, address, MMU_WRITE_BACK);
 		(*cleaned)++;
 	}
 out:

@@ -1338,7 +1338,7 @@ discard:
 out_unmap:
 	pte_unmap_unlock(pte, ptl);
 	if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK))
-		mmu_notifier_invalidate_page(mm, address);
+		mmu_notifier_invalidate_page(mm, address, MMU_MIGRATE);
 out:
 	return ret;
 

virt/kvm/kvm_main.c +8 -4

index f202c40..d0b1060 100644
@@ -260,7 +260,8 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 
 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
-					     unsigned long address)
+					     unsigned long address,
+					     enum mmu_event event)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush, idx;

@@ -302,7 +303,8 @@ static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 					struct mm_struct *mm,
 					unsigned long address,
-					pte_t pte)
+					pte_t pte,
+					enum mmu_event event)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int idx;

@@ -318,7 +320,8 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 						    struct mm_struct *mm,
 						    unsigned long start,
-						    unsigned long end)
+						    unsigned long end,
+						    enum mmu_event event)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush = 0, idx;

@@ -344,7 +347,8 @@ static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 						  struct mm_struct *mm,
 						  unsigned long start,
-						  unsigned long end)
+						  unsigned long end,
+						  enum mmu_event event)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 

Signature · 1 line

1.9.3

Quoted 20 lines · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

The event information will be useful for new user of mmu_notifier API. The event argument differentiate between a vma disappearing, a page being write protected or simply a page being unmaped. This allow new user to take different path for different event for instance on unmap the resource used to track a vma are still valid and should stay around. While if the event is saying that a vma is being destroy it means that any resources used to track this vma can be free.

Changed since v1:
- renamed action into event (updated commit message too).
- simplified the event names and clarified their usage also documenting what exceptation the listener can have in respect to each event.

Changed since v2:
- Avoid crazy name.
- Do not move code that do not need to move.

Quoted 2 lines

Changed since v3:
- Separate hugue page split from mlock/munlock and softdirty.

Do we care about fixing up patch comments? If so:

s/hugue/huge/

include/linux/mmu_notifier.h line 66

Quoted 1 line

include/linux/mmu_notifier.h +1 -0

+	MMU_HSPLIT,

Let's rename MMU_HSPLIT to one of the following, take your pick:

MMU_HUGE_PAGE_SPLIT (too long, but you can't possibly misunderstand it) MMU_PAGE_SPLIT (my favorite: only huge pages are ever split, so it works) MMU_HUGE_SPLIT (ugly, but still hard to misunderstand)

include/linux/mmu_notifier.h line 67

Quoted 2 lines

include/linux/mmu_notifier.h +1 -0

+	MMU_ISDIRTY,

This MMU_ISDIRTY seems like a problem to me. First of all, it looks
backwards: the only place that invokes it is the clear_refs_write() routine, for the soft-dirty tracking feature. And in that case, the pages are *not* being made dirty! Rather, the kernel is actually making the pages non-writable, in order to be able to trap the subsequent page fault and figure out if the page is in active use.

So, given that there is only one call site, and that call site should actually be setting MMU_WRITE_PROTECT instead (I think), let's just delete MMU_ISDIRTY.

Come to think about it, there is no callback possible for "a page became dirty", anyway. Because the dirty and accessed bits are actually set by the hardware, and software is generally unable to know the current state. So MMU_ISDIRTY just seems inappropriate to me, across the board.

I'll take a look at the corresponding HMM_ISDIRTY, too.

include/linux/mmu_notifier.h line 69

Quoted 3 lines

include/linux/mmu_notifier.h +2 -0

+	MMU_MIGRATE,
+	MMU_MPROT,

The MMU_PROT also looks questionable. Short answer: probably better to read the protection, and pass either MMU_WRITE_PROTECT, MMU_READ_WRITE (that's a new item, of course), or MMU_UNMAP.

Here's why: the call site knows the protection, but by the time it filters down to HMM (in later patches), that information is lost, and HMM ends up doing (ouch!) another find_vma() call in order to retrieve it--and then translates it into only three possible things:

// hmm_mmu_mprot_to_etype() sets one of these:

HMM_MUNMAP
HMM_WRITE_PROTECT
HMM_NONE

include/linux/mmu_notifier.h line 70

Quoted 3 lines

include/linux/mmu_notifier.h +1 -0

+	MMU_MUNLOCK,

I think MMU_UNLOCK would be clearer. We already know the scope, so the extra "M" isn't adding anything.

include/linux/mmu_notifier.h line 71

Quoted 2 lines

include/linux/mmu_notifier.h +1 -0

+	MMU_MUNMAP,

Same thing here: MMU_UNMAP seems better.

include/linux/mmu_notifier.h line 73

Quoted 3 lines

include/linux/mmu_notifier.h +2 -0

+	MMU_WRITE_BACK,
+	MMU_WRITE_PROTECT,

We may have to add MMU_READ_WRITE (and maybe another one, I haven't bottomed out on that), if you agree with the above approach of always sending a precise event, instead of "protection changed".

Quoted 22 lines · John Hubbard

Quoted 20 lines · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

The event information will be useful for new user of mmu_notifier API. The event argument differentiate between a vma disappearing, a page being write protected or simply a page being unmaped. This allow new user to take different path for different event for instance on unmap the resource used to track a vma are still valid and should stay around. While if the event is saying that a vma is being destroy it means that any resources used to track this vma can be free.

Changed since v1:
- renamed action into event (updated commit message too).
- simplified the event names and clarified their usage also documenting what exceptation the listener can have in respect to each event.

Changed since v2:
- Avoid crazy name.
- Do not move code that do not need to move.

Quoted 6 lines

Quoted 2 lines

Changed since v3:
- Separate hugue page split from mlock/munlock and softdirty.

Do we care about fixing up patch comments? If so:

s/hugue/huge/

I am noting them down and will go over them.

[...]

Quoted 7 lines

include/linux/mmu_notifier.h line 66

Quoted 1 line

include/linux/mmu_notifier.h +1 -0

+	MMU_HSPLIT,

Let's rename MMU_HSPLIT to one of the following, take your pick:

MMU_HUGE_PAGE_SPLIT (too long, but you can't possibly misunderstand it) MMU_PAGE_SPLIT (my favorite: only huge pages are ever split, so it works) MMU_HUGE_SPLIT (ugly, but still hard to misunderstand)

I will go with MMU_HUGE_PAGE_SPLIT

[...]

Quoted 14 lines

include/linux/mmu_notifier.h line 67

Quoted 2 lines

include/linux/mmu_notifier.h +1 -0

+	MMU_ISDIRTY,

This MMU_ISDIRTY seems like a problem to me. First of all, it looks
backwards: the only place that invokes it is the clear_refs_write() routine, for the soft-dirty tracking feature. And in that case, the pages are *not* being made dirty! Rather, the kernel is actually making the pages non-writable, in order to be able to trap the subsequent page fault and figure out if the page is in active use.

So, given that there is only one call site, and that call site should actually be setting MMU_WRITE_PROTECT instead (I think), let's just delete MMU_ISDIRTY.

Quoted 6 lines

Come to think about it, there is no callback possible for "a page became dirty", anyway. Because the dirty and accessed bits are actually set by the hardware, and software is generally unable to know the current state. So MMU_ISDIRTY just seems inappropriate to me, across the board.

I'll take a look at the corresponding HMM_ISDIRTY, too.

Ok i need to rename that one to CLEAR_SOFT_DIRTY, the idea is that for HMM i would rather not write protect the memory for the device and just rely on the regular and conservative dirtying of page. The soft dirty is really for migrating a process where you first clear the soft dirty bit, then copy memory while process still running, then freeze process an only copy memory that was dirtied since first copy. Point being that adding soft dirty to HMM is something that can be done down the road. We should have enough bit inside the device page table for that.

Quoted 14 lines

include/linux/mmu_notifier.h line 69

Quoted 4 lines

include/linux/mmu_notifier.h +2 -0

+	MMU_MIGRATE,
+	MMU_MPROT,

The MMU_PROT also looks questionable. Short answer: probably better to read the protection, and pass either MMU_WRITE_PROTECT, MMU_READ_WRITE (that's a new item, of course), or MMU_UNMAP.

Here's why: the call site knows the protection, but by the time it filters down to HMM (in later patches), that information is lost, and HMM ends up doing (ouch!) another find_vma() call in order to retrieve it--and then translates it into only three possible things:

Quoted 5 lines

// hmm_mmu_mprot_to_etype() sets one of these:

HMM_MUNMAP
HMM_WRITE_PROTECT
HMM_NONE

Linus complained of my previous version where i differenciated the kind of protection change that was happening, hence why i only pass down mprot.

Quoted 6 lines

include/linux/mmu_notifier.h line 70

Quoted 3 lines

include/linux/mmu_notifier.h +1 -0

+	MMU_MUNLOCK,

I think MMU_UNLOCK would be clearer. We already know the scope, so the extra "M" isn't adding anything.

I named it that way so it matches syscall name munlock(). I think it is clearer to use MUNLOCK, or maybe SYSCALL_MUNLOCK

Quoted 1 line

Quoted 3 lines

include/linux/mmu_notifier.h line 71

Quoted 1 line

include/linux/mmu_notifier.h +1 -0

+	MMU_MUNMAP,

Same thing here: MMU_UNMAP seems better.

Well same idea here.

Quoted 9 lines

include/linux/mmu_notifier.h line 73

Quoted 5 lines

include/linux/mmu_notifier.h +2 -0

+	MMU_WRITE_BACK,
+	MMU_WRITE_PROTECT,

We may have to add MMU_READ_WRITE (and maybe another one, I haven't bottomed out on that), if you agree with the above approach of always sending a precise event, instead of "protection changed".

I think Linus point made sense last time, but i would need to read again the thread. The idea of that patch is really to provide context information on what kind of CPU page table changes is happening and why.

In that respect i should probably change MMU_WRITE_PROTECT to MMU_KSM_WRITE_PROTECT.

Cheers,
Jerome

Quoted 3 lines · Jerome Glisse

Quoted 2 lines · John Hubbard

Quoted 1 line · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

Quoted 20 lines

Quoted 13 lines

include/linux/mmu_notifier.h line 67

Quoted 1 line

include/linux/mmu_notifier.h +1 -0

+	MMU_ISDIRTY,

This MMU_ISDIRTY seems like a problem to me. First of all, it looks
backwards: the only place that invokes it is the clear_refs_write() routine, for the soft-dirty tracking feature. And in that case, the pages are *not* being made dirty! Rather, the kernel is actually making the pages non-writable, in order to be able to trap the subsequent page fault and figure out if the page is in active use.

So, given that there is only one call site, and that call site should actually be setting MMU_WRITE_PROTECT instead (I think), let's just delete MMU_ISDIRTY.

Quoted 6 lines

Come to think about it, there is no callback possible for "a page became dirty", anyway. Because the dirty and accessed bits are actually set by the hardware, and software is generally unable to know the current state. So MMU_ISDIRTY just seems inappropriate to me, across the board.

I'll take a look at the corresponding HMM_ISDIRTY, too.

Quoted 10 lines

Ok i need to rename that one to CLEAR_SOFT_DIRTY, the idea is that for HMM i would rather not write protect the memory for the device and just rely on the regular and conservative dirtying of page. The soft dirty is really for migrating a process where you first clear the soft dirty bit, then copy memory while process still running, then freeze process an only copy memory that was dirtied since first copy. Point being that adding soft dirty to HMM is something that can be done down the road. We should have enough bit inside the device page table for that.

Yes, I think renaming it to CLEAR_SOFT_DIRTY will definitely allow more accurate behavior in response to these events.

Looking ahead, a couple things:

1. This mechanism is also used for general memory utilization tracking (I see that Vladimir DavyDov has an "idle memory tracking" proposal that assumes this works, for example: https://lwn.net/Articles/642202/ and
https://lkml.org/lkml/2015/5/12/449).

2. It seems hard to avoid the need to eventually just write protect the page, whether it is on the CPU or the remote device, if things like device drivers or user space need to track write accesses to a virtual address. Either you write protect the page, and trap the page faults, or you wait until later and read the dirty bit (indirectly, via something like unmap_mapping_range). Or did you have something else in mind?

Anyway, none of that needs to hold up this part of the patchset, because the renaming fixes things up for the future code to do the right thing.

Quoted 9 lines

Quoted 3 lines

We may have to add MMU_READ_WRITE (and maybe another one, I haven't bottomed out on that), if you agree with the above approach of always sending a precise event, instead of "protection changed".

I think Linus point made sense last time, but i would need to read again the thread. The idea of that patch is really to provide context information on what kind of CPU page table changes is happening and why.

Shoot, I tried to find that conversation, but my search foo is too weak. If you have a link to that thread, I'd appreciate it, so I can refresh my memory.

I was hoping to re-read it and see if anything has changed. It's not really a huge problem to call find_vma() again, but I do want to be sure that there's a good reason for doing so.

Otherwise, I'll just rely on your memory that Linus preferred your current approach, and call it good, then.

Quoted 4 lines · John Hubbard

Quoted 3 lines · Jerome Glisse

Quoted 2 lines · John Hubbard

Quoted 1 line · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

[...]

Quoted 18 lines

Quoted 9 lines

Quoted 3 lines

We may have to add MMU_READ_WRITE (and maybe another one, I haven't bottomed out on that), if you agree with the above approach of always sending a precise event, instead of "protection changed".

I think Linus point made sense last time, but i would need to read again the thread. The idea of that patch is really to provide context information on what kind of CPU page table changes is happening and why.

Shoot, I tried to find that conversation, but my search foo is too weak. If you have a link to that thread, I'd appreciate it, so I can refresh my memory.

I was hoping to re-read it and see if anything has changed. It's not really a huge problem to call find_vma() again, but I do want to be sure that there's a good reason for doing so.

Quoted 2 lines

Otherwise, I'll just rely on your memory that Linus preferred your current approach, and call it good, then.

http://lkml.iu.edu/hypermail/linux/kernel/1406.3/04880.html

I am working on doing some of the changes discussed so far, i will push my tree to git://people.freedesktop.org/~glisse/linux hmm branch once i am done.

Cheers,
Jerome

From: Jérôme Glisse <[email protected]>

The mmu_notifier_invalidate_range_start() and mmu_notifier_invalidate_range_end() can be considered as forming an "atomic" section for the cpu page table update point of view. Between this two function the cpu page table content is unreliable for the address range being invalidated.

Current user such as kvm need to know when they can trust the content of the cpu page table. This becomes even more important to new users of the mmu_notifier api (such as HMM or ODP).

This patch use a structure define at all call site to invalidate_range_start() that is added to a list for the duration of the invalidation. It adds two new helpers to allow querying if a range is being invalidated or to wait for a range to become valid.

For proper synchronization, user must block new range invalidation from inside there invalidate_range_start() callback, before calling the helper functions. Otherwise there is no garanty that a new range invalidation will not be added after the call to the helper function to query for existing range.

Changed since v1:
- Fix a possible deadlock in mmu_notifier_range_wait_valid()

Changed since v2:
- Add the range to invalid range list before calling ->range_start().
- Del the range from invalid range list after calling ->range_end().
- Remove useless list initialization.

Quoted 7 lines · [email protected] writes:

From: Jerome Glisse <[email protected]>

The mmu_notifier_invalidate_range_start() and mmu_notifier_invalidate_range_end() can be considered as forming an "atomic" section for the cpu page table update point of view. Between this two function the cpu page table content is unreliable for the address range being invalidated.

Quoted 3 lines

Current user such as kvm need to know when they can trust the content of the cpu page table. This becomes even more important to new users of the mmu_notifier api (such as HMM or ODP).

I don't see kvm using the new APIs in this patch. Also what is that HMM use this for, to protect walking of mirror page table ?. I am sure you are covering that in the later patches. May be you may want to mention the details here too.

From: JA(C)rA'me Glisse <[email protected]>

The mmu_notifier_invalidate_range_start() and mmu_notifier_invalidate_range_end() can be considered as forming an "atomic" section for the cpu page table update point of view. Between this two function the cpu page table content is unreliable for the address range being invalidated.

Current user such as kvm need to know when they can trust the content of the cpu page table. This becomes even more important to new users of the mmu_notifier api (such as HMM or ODP).

This patch use a structure define at all call site to invalidate_range_start() that is added to a list for the duration of the invalidation. It adds two new helpers to allow querying if a range is being invalidated or to wait for a range to become valid.

For proper synchronization, user must block new range invalidation from inside there invalidate_range_start() callback, before calling the helper functions. Otherwise there is no garanty that a new range invalidation will not be added after the call to the helper function to query for existing range.

Changed since v1:
- Fix a possible deadlock in mmu_notifier_range_wait_valid()

Changed since v2:
- Add the range to invalid range list before calling ->range_start().
- Del the range from invalid range list after calling ->range_end().
- Remove useless list initialization.

Signed-off-by JA(C)rA'me Glisse <[email protected]>

Reviewed-by Rik van Riel <[email protected]>

Reviewed-by Haggai Eran <[email protected]>

---
 drivers/gpu/drm/i915/i915_gem_userptr.c |  9 ++--
 drivers/gpu/drm/radeon/radeon_mn.c      | 14 +++---
 drivers/infiniband/core/umem_odp.c      | 16 +++----
 drivers/misc/sgi-gru/grutlbpurge.c      | 15 +++----
 drivers/xen/gntdev.c                    | 15 ++++---
 fs/proc/task_mmu.c                      | 11 +++--
 include/linux/mmu_notifier.h            | 55 ++++++++++++-----------
 kernel/events/uprobes.c                 | 13 +++---
 mm/huge_memory.c                        | 78 ++++++++++++++------------------
 mm/hugetlb.c                            | 55 ++++++++++++-----------
 mm/ksm.c                                | 28 +++++-------
 mm/madvise.c                            | 20 ++++-----
 mm/memory.c                             | 72 +++++++++++++++++-------------
 mm/migrate.c                            | 36 +++++++--------
 mm/mmu_notifier.c                       | 79 ++++++++++++++++++++++++++++-----
 mm/mprotect.c                           | 18 ++++----
 mm/mremap.c                             | 14 +++---
 virt/kvm/kvm_main.c                     | 10 ++---
 18 files changed, 302 insertions(+), 256 deletions(-)

drivers/gpu/drm/i915/i915_gem_userptr.c +0 -2

index 452e9b1..80fe72a 100644
@@ -131,16 +131,15 @@ restart:
 
 static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 						       struct mm_struct *mm,
-						       unsigned long start,
-						       unsigned long end,

drivers/gpu/drm/i915/i915_gem_userptr.c +18 -8

+	spin_unlock(&mm->mmu_notifier_mm->lock);
+
 	id = srcu_read_lock(&srcu);
 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
 		if (mn->ops->invalidate_range_start)
-			mn->ops->invalidate_range_start(mn, mm, start,
-							end, event);
+			mn->ops->invalidate_range_start(mn, mm, range);
 	}
 	srcu_read_unlock(&srcu, id);
 }
 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
 
 void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
-					 unsigned long start,
-					 unsigned long end,
-					 enum mmu_event event)
+					 struct mmu_notifier_range *range)
 {
 	struct mmu_notifier *mn;
 	int id;

@@ -211,12 +211,23 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 		 * (besides the pointer check).
 		 */
 		if (mn->ops->invalidate_range)
-			mn->ops->invalidate_range(mn, mm, start, end);
+			mn->ops->invalidate_range(mn, mm,
+						  range->start, range->end);
 		if (mn->ops->invalidate_range_end)
-			mn->ops->invalidate_range_end(mn, mm, start,
-						      end, event);
+			mn->ops->invalidate_range_end(mn, mm, range);
 	}
 	srcu_read_unlock(&srcu, id);
+
+	spin_lock(&mm->mmu_notifier_mm->lock);
+	list_del_init(&range->list);
+	mm->mmu_notifier_mm->nranges--;
+	spin_unlock(&mm->mmu_notifier_mm->lock);
+
+	/*
+	 * Wakeup after callback so they can do their job before any of the
+	 * waiters resume.
+	 */
+	wake_up(&mm->mmu_notifier_mm->wait_queue);
 }
 EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
 

drivers/gpu/drm/i915/i915_gem_userptr.c +34 -0

+	}
+	return true;
+}
+
+bool mmu_notifier_range_is_valid(struct mm_struct *mm,
+				 unsigned long start,
+				 unsigned long end)
+{
+	bool valid;
+
+	spin_lock(&mm->mmu_notifier_mm->lock);
+	valid = mmu_notifier_range_is_valid_locked(mm, start, end);
+	spin_unlock(&mm->mmu_notifier_mm->lock);
+	return valid;
+}
+EXPORT_SYMBOL_GPL(mmu_notifier_range_is_valid);
+
+void mmu_notifier_range_wait_valid(struct mm_struct *mm,
+				   unsigned long start,
+				   unsigned long end)
+{
+	spin_lock(&mm->mmu_notifier_mm->lock);
+	while (!mmu_notifier_range_is_valid_locked(mm, start, end)) {
+		int nranges = mm->mmu_notifier_mm->nranges;
+
+		spin_unlock(&mm->mmu_notifier_mm->lock);
+		wait_event(mm->mmu_notifier_mm->wait_queue,
+			   nranges != mm->mmu_notifier_mm->nranges);
+		spin_lock(&mm->mmu_notifier_mm->lock);
+	}
+	spin_unlock(&mm->mmu_notifier_mm->lock);
+}
+EXPORT_SYMBOL_GPL(mmu_notifier_range_wait_valid);
+
 static int do_mmu_notifier_register(struct mmu_notifier *mn,
 				    struct mm_struct *mm,
 				    int take_mmap_sem)

Quoted 16 lines · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

The mmu_notifier_invalidate_range_start() and mmu_notifier_invalidate_range_end() can be considered as forming an "atomic" section for the cpu page table update point of view. Between this two function the cpu page table content is unreliable for the address range being invalidated.

Current user such as kvm need to know when they can trust the content of the cpu page table. This becomes even more important to new users of the mmu_notifier api (such as HMM or ODP).

This patch use a structure define at all call site to invalidate_range_start() that is added to a list for the duration of the invalidation. It adds two new helpers to allow querying if a range is being invalidated or to wait for a range to become valid.

Quoted 4 lines

For proper synchronization, user must block new range invalidation from inside there invalidate_range_start() callback, before calling the helper functions. Otherwise there is no garanty that a new range invalidation will not be added after the call to the helper function to query for existing range.

Hi Jerome,

Most of this information will make nice block comments for the new helper routines. I can help tighten up the writing slightly, but first:

Question: in hmm.c's hmm_notifier_invalidate function (looking at the entire patchset, for a moment), I don't see any blocking of new range invalidations, even though you point out, above, that this is required. Am I missing it, and if so, where should I be looking instead?

[...]

Quoted 7 lines

mm/mmu_notifier.c +1 -1

-					   enum mmu_event event)
+					   struct mmu_notifier_range *range)
 
 {
 	struct mmu_notifier *mn;
 	int id;

mm/mmu_notifier.c void __mmu_notifier_invalidate_page(struct mm_struct *mm, · line 185

Quoted 3 lines

mm/mmu_notifier.c +3 -0

+	spin_lock(&mm->mmu_notifier_mm->lock);
+	list_add_tail(&range->list, &mm->mmu_notifier_mm->ranges);
+	mm->mmu_notifier_mm->nranges++;

Is this missing a call to wake_up(&mm->mmu_notifier_mm->wait_queue)? If not, then it would be helpful to explain why that's only required for nranges--, and not for the nranges++ case. The helper routine is merely waiting for nranges to *change*, not looking for greater than or less than.

mm/mmu_notifier.c void __mmu_notifier_invalidate_range(struct mm_struct *mm, · line 251

Quoted 3 lines

mm/mmu_notifier.c +3 -0

+static bool mmu_notifier_range_is_valid_locked(struct mm_struct *mm,
+					       unsigned long start,
+					       unsigned long end)

This routine is named "_range_is_valid_", but it takes in an implicit range (start, end), and also a list of ranges (buried in mm), and so it's a little confusing. I'd like to consider *maybe* changing either the name, or the args (range* instead of start, end?), or something.

Could you please say a few words about the intent of this routine, to get us started there?

Quoted 25 lines · John Hubbard

Quoted 16 lines · On Thu, 21 May 2015, [email protected] wrote:

From: Jerome Glisse <[email protected]>

The mmu_notifier_invalidate_range_start() and mmu_notifier_invalidate_range_end() can be considered as forming an "atomic" section for the cpu page table update point of view. Between this two function the cpu page table content is unreliable for the address range being invalidated.

Current user such as kvm need to know when they can trust the content of the cpu page table. This becomes even more important to new users of the mmu_notifier api (such as HMM or ODP).

This patch use a structure define at all call site to invalidate_range_start() that is added to a list for the duration of the invalidation. It adds two new helpers to allow querying if a range is being invalidated or to wait for a range to become valid.

Quoted 4 lines

For proper synchronization, user must block new range invalidation from inside there invalidate_range_start() callback, before calling the helper functions. Otherwise there is no garanty that a new range invalidation will not be added after the call to the helper function to query for existing range.

Hi Jerome,

Quoted 7 lines

Most of this information will make nice block comments for the new helper routines. I can help tighten up the writing slightly, but first:

Question: in hmm.c's hmm_notifier_invalidate function (looking at the entire patchset, for a moment), I don't see any blocking of new range invalidations, even though you point out, above, that this is required. Am I missing it, and if so, where should I be looking instead?

This is a 2 sided synchronization:

- hmm_device_fault_start() will wait for active invalidation that conflict to be done
- hmm_wait_device_fault() will block new invalidation until active fault that conflict back off.

Quoted 10 lines

[...]

Quoted 7 lines

mm/mmu_notifier.c +1 -1

-					   enum mmu_event event)
+					   struct mmu_notifier_range *range)
 
 {
 	struct mmu_notifier *mn;
 	int id;

Quoted 10 lines

mm/mmu_notifier.c void __mmu_notifier_invalidate_page(struct mm_struct *mm, · line 185

Quoted 3 lines

mm/mmu_notifier.c +3 -0

+	spin_lock(&mm->mmu_notifier_mm->lock);
+	list_add_tail(&range->list, &mm->mmu_notifier_mm->ranges);
+	mm->mmu_notifier_mm->nranges++;

Is this missing a call to wake_up(&mm->mmu_notifier_mm->wait_queue)? If not, then it would be helpful to explain why that's only required for nranges--, and not for the nranges++ case. The helper routine is merely waiting for nranges to *change*, not looking for greater than or less than.

This is on purpose, as the waiting side only wait for active invalidation to be done ie for mm->mmu_notifier_mm->nranges-- so there is no reasons to wake up when a new invalidation is starting. Also the test need to be a not equal because other non conflicting range might be added/removed meaning that wait might finish even if mm->mmu_notifier_mm->nranges > saved_nranges.

[...]

Quoted 10 lines

mm/mmu_notifier.c void __mmu_notifier_invalidate_range(struct mm_struct *mm, · line 251

Quoted 3 lines

mm/mmu_notifier.c +3 -0

+static bool mmu_notifier_range_is_valid_locked(struct mm_struct *mm,
+					       unsigned long start,
+					       unsigned long end)

This routine is named "_range_is_valid_", but it takes in an implicit range (start, end), and also a list of ranges (buried in mm), and so it's a little confusing. I'd like to consider *maybe* changing either the name, or the args (range* instead of start, end?), or something.

Quoted 2 lines

Could you please say a few words about the intent of this routine, to get us started there?

It is just the same as mmu_notifier_range_is_valid() but it expects locks to be taken. This is for the benefit of mmu_notifier_range_wait_valid() which need to test if a range is valid (ie no conflicting invalidation) or not. I added a comment to explain this 3 function and to explain how the 2 publics helper needs to be use.

Cheers,
Jerome

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

From: Jérôme Glisse <[email protected]>

Listener of mm event might not have easy way to get the struct page behind and address invalidated with mmu_notifier_invalidate_page() function as this happens after the cpu page table have been clear/ updated. This happens for instance if the listener is storing a dma mapping inside its secondary page table. To avoid complex reverse dma mapping lookup just pass along a pointer to the page being invalidated.

Quoted 9 lines · [email protected] writes:

From: Jerome Glisse <[email protected]>

Listener of mm event might not have easy way to get the struct page behind and address invalidated with mmu_notifier_invalidate_page() function as this happens after the cpu page table have been clear/ updated. This happens for instance if the listener is storing a dma mapping inside its secondary page table. To avoid complex reverse dma mapping lookup just pass along a pointer to the page being invalidated.

.....

include/linux/mmu_notifier.h struct mmu_notifier_ops { · line 177

Quoted 11 lines

include/linux/mmu_notifier.h +1 -0

index ada3ed1..283ad26 100644
@@ -172,6 +172,7 @@ struct mmu_notifier_ops {
 	void (*invalidate_page)(struct mmu_notifier *mn,
 				struct mm_struct *mm,
 				unsigned long address,
+				struct page *page,
 				enum mmu_event event);
 

How do we handle this w.r.t invalidate_range ?

From: JA(C)rA'me Glisse <[email protected]>

Listener of mm event might not have easy way to get the struct page behind and address invalidated with mmu_notifier_invalidate_page()

function as this happens after the cpu page table have been clear/ updated. This happens for instance if the listener is storing a dma mapping inside its secondary page table. To avoid complex reverse dma mapping lookup just pass along a pointer to the page being invalidated.

Signed-off-by JA(C)rA'me Glisse <[email protected]>

---
 drivers/infiniband/core/umem_odp.c | 1 +
 drivers/iommu/amd_iommu_v2.c       | 1 +
 drivers/misc/sgi-gru/grutlbpurge.c | 1 +
 drivers/xen/gntdev.c               | 1 +
 include/linux/mmu_notifier.h       | 6 +++++-
 mm/mmu_notifier.c                  | 3 ++-
 mm/rmap.c                          | 4 ++--
 virt/kvm/kvm_main.c                | 1 +
 8 files changed, 14 insertions(+), 4 deletions(-)

drivers/infiniband/core/umem_odp.c +1 -0

index 8f7f845..d10dd88 100644
@@ -166,6 +166,7 @@ static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
 static void ib_umem_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long address,
+					     struct page *page,
 					     enum mmu_event event)
 {
 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);

drivers/iommu/amd_iommu_v2.c +1 -0

index 4aa4de6..de3c540 100644
@@ -385,6 +385,7 @@ static int mn_clear_flush_young(struct mmu_notifier *mn,
 static void mn_invalidate_page(struct mmu_notifier *mn,
 			       struct mm_struct *mm,
 			       unsigned long address,
+			       struct page *page,
 			       enum mmu_event event)
 {
 	__mn_flush_page(mn, address);

drivers/misc/sgi-gru/grutlbpurge.c +1 -0

index 44b41b7..c7659b76 100644
@@ -250,6 +250,7 @@ static void gru_invalidate_range_end(struct mmu_notifier *mn,
 
 static void gru_invalidate_page(struct mmu_notifier *mn, struct mm_struct *mm,
 				unsigned long address,
+				struct page *page,
 				enum mmu_event event)
 {
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,

drivers/xen/gntdev.c +1 -0

index 0e8aa12..90693ce 100644
@@ -485,6 +485,7 @@ static void mn_invl_range_start(struct mmu_notifier *mn,
 static void mn_invl_page(struct mmu_notifier *mn,
 			 struct mm_struct *mm,
 			 unsigned long address,
+			 struct page *page,
 			 enum mmu_event event)
 {
 	struct mmu_notifier_range range;

include/linux/mmu_notifier.h +5 -1

index ada3ed1..283ad26 100644
@@ -172,6 +172,7 @@ struct mmu_notifier_ops {
 	void (*invalidate_page)(struct mmu_notifier *mn,
 				struct mm_struct *mm,
 				unsigned long address,
+				struct page *page,
 				enum mmu_event event);
 
 	/*

@@ -290,6 +291,7 @@ extern void __mmu_notifier_change_pte(struct mm_struct *mm,
 				      enum mmu_event event);
 extern void __mmu_notifier_invalidate_page(struct mm_struct *mm,
 					  unsigned long address,
+					  struct page *page,
 					  enum mmu_event event);
 extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
 						  struct mmu_notifier_range *range);

@@ -338,10 +340,11 @@ static inline void mmu_notifier_change_pte(struct mm_struct *mm,
 
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
 						unsigned long address,
+						struct page *page,
 						enum mmu_event event)
 {
 	if (mm_has_notifiers(mm))
-		__mmu_notifier_invalidate_page(mm, address, event);
+		__mmu_notifier_invalidate_page(mm, address, page, event);
 }
 
 static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,

@@ -492,6 +495,7 @@ static inline void mmu_notifier_change_pte(struct mm_struct *mm,
 
 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
 						unsigned long address,
+						struct page *page,
 						enum mmu_event event)
 {
 }

mm/mmu_notifier.c +2 -1

index 294ebc4..2ff6d43 100644
@@ -160,6 +160,7 @@ void __mmu_notifier_change_pte(struct mm_struct *mm,
 
 void __mmu_notifier_invalidate_page(struct mm_struct *mm,
 				    unsigned long address,
+				    struct page *page,
 				    enum mmu_event event)
 {
 	struct mmu_notifier *mn;

@@ -168,7 +169,7 @@ void __mmu_notifier_invalidate_page(struct mm_struct *mm,
 	id = srcu_read_lock(&srcu);
 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
 		if (mn->ops->invalidate_page)
-			mn->ops->invalidate_page(mn, mm, address, event);
+			mn->ops->invalidate_page(mn, mm, address, page, event);
 	}
 	srcu_read_unlock(&srcu, id);
 }

mm/rmap.c +2 -2

index 74c51e0..4563edc 100644
@@ -915,7 +915,7 @@ static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
 	pte_unmap_unlock(pte, ptl);
 
 	if (ret) {
-		mmu_notifier_invalidate_page(mm, address, MMU_WRITE_BACK);
+		mmu_notifier_invalidate_page(mm, address, page, MMU_WRITE_BACK);
 		(*cleaned)++;
 	}
 out:

@@ -1338,7 +1338,7 @@ discard:
 out_unmap:
 	pte_unmap_unlock(pte, ptl);
 	if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK))
-		mmu_notifier_invalidate_page(mm, address, MMU_MIGRATE);
+		mmu_notifier_invalidate_page(mm, address, page, MMU_MIGRATE);
 out:
 	return ret;
 

virt/kvm/kvm_main.c +1 -0

index 6177c56..62978ed 100644
@@ -261,6 +261,7 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long address,
+					     struct page *page,
 					     enum mmu_event event)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);

Signature · 1 line

1.9.3