Skip to contentSkip to conversation

Removing the lru_cache_add() wrapper

PATCH Applied Vishal Moola (Oracle) 6 participants 20 messages Nov 1, 2022 – Nov 29, 2022

Subsystems (10)
Memory management VFS MM core MGLRU +6 more subsystems
lore
Share & tools
lore
#0 lore raw

cover Removing the lru_cache_add() wrapper

This patchset replaces all calls of lru_cache_add() with the folio
equivalent: folio_add_lru(). This is allows us to get rid of the wrapper The series passes xfstests and the userfaultfd selftests.

I haven't been able to hit the fuse_try_move_page() function, so I haven't been able to test those changes. Since replace_page_cache_page() is only called by fuse_try_move_page() I haven't been able to test that as well. Testing and review of both would be beneficial.

Vishal Moola (Oracle) (5):
filemap: Convert replace_page_cache_page() to
replace_page_cache_folio()
fuse: Convert fuse_try_move_page() to use folios
userfualtfd: Replace lru_cache functions with folio_add functions
khugepage: Replace lru_cache_add() with folio_add_lru()
folio-compat: Remove lru_cache_add()

 fs/fuse/dev.c           | 55 +++++++++++++++++++++--------------------
 include/linux/pagemap.h |  2 +-
 include/linux/swap.h    |  1 -
 mm/filemap.c            | 52 +++++++++++++++++++-------------------
 mm/folio-compat.c       |  6 -----
 mm/khugepaged.c         | 11 ++++++---
 mm/truncate.c           |  2 +-
 mm/userfaultfd.c        |  6 +++--
 mm/workingset.c         |  2 +-
 9 files changed, 67 insertions(+), 70 deletions(-)
Signature · 1 line
2.38.1
#1 lore raw

PATCH 1/5 filemap: Convert replace_page_cache_page() to replace_page_cache_folio()

Eliminates 7 calls to compound_head().

---
 fs/fuse/dev.c           |  2 +-
 include/linux/pagemap.h |  2 +-
 mm/filemap.c            | 52 ++++++++++++++++++++---------------------
 3 files changed, 27 insertions(+), 29 deletions(-)
fs/fuse/dev.c +1 -1
index b4a6e0a1b945..26817a2db463 100644
@@ -837,7 +837,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 	if (WARN_ON(PageMlocked(oldpage)))
 		goto out_fallback_unlock;
 
-	replace_page_cache_page(oldpage, newpage);
+	replace_page_cache_folio(page_folio(oldpage), page_folio(newpage));
 
 	get_page(newpage);
 
include/linux/pagemap.h +1 -1
index bbccb4044222..275810697d71 100644
@@ -1104,7 +1104,7 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio,
 void filemap_remove_folio(struct folio *folio);
 void delete_from_page_cache(struct page *page);
 void __filemap_remove_folio(struct folio *folio, void *shadow);
-void replace_page_cache_page(struct page *old, struct page *new);
+void replace_page_cache_folio(struct folio *old, struct folio *new);
 void delete_from_page_cache_batch(struct address_space *mapping,
 				  struct folio_batch *fbatch);
 int try_to_release_page(struct page *page, gfp_t gfp);
mm/filemap.c +25 -27

81 lines not on this page · raw message

Signature · 1 line
2.38.1
#3 lore raw

PATCH 2/5 fuse: Convert fuse_try_move_page() to use folios

In reply to Vishal Moola (Oracle)

Converts the function to try to move folios instead of pages. Also converts fuse_check_page() to fuse_get_folio() since this is its only caller. This change removes 15 calls to compound_head().

---
 fs/fuse/dev.c | 55 ++++++++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)
fs/fuse/dev.c +28 -27
index 26817a2db463..204c332cd343 100644
@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 	return ncpy;
 }
 
-static int fuse_check_page(struct page *page)
+static int fuse_check_folio(struct folio *folio)
 {
-	if (page_mapcount(page) ||
-	    page->mapping != NULL ||
-	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
+	if (folio_mapped(folio) ||
+	    folio->mapping != NULL ||
+	    (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &
 	     ~(1 << PG_locked |
 	       1 << PG_referenced |
 	       1 << PG_uptodate |
@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page)
 	       1 << PG_reclaim |
 	       1 << PG_waiters |
 	       LRU_GEN_MASK | LRU_REFS_MASK))) {
-		dump_page(page, "fuse: trying to steal weird page");
+		dump_page(&folio->page, "fuse: trying to steal weird page");
 		return 1;
 	}
 	return 0;
@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)
 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 {
 	int err;
-	struct page *oldpage = *pagep;
-	struct page *newpage;
+	struct folio *oldfolio = page_folio(*pagep);
+	struct folio *newfolio;
 	struct pipe_buffer *buf = cs->pipebufs;
 
-	get_page(oldpage);
+	folio_get(oldfolio);
 	err = unlock_request(cs->req);
 	if (err)
 		goto out_put_old;
@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 	if (!pipe_buf_try_steal(cs->pipe, buf))
 		goto out_fallback;
 
-	newpage = buf->page;
+	newfolio = page_folio(buf->page);
 
-	if (!PageUptodate(newpage))
-		SetPageUptodate(newpage);
+	if (!folio_test_uptodate(newfolio))
+		folio_mark_uptodate(newfolio);
 
-	ClearPageMappedToDisk(newpage);
+	folio_clear_mappedtodisk(newfolio);
 
-	if (fuse_check_page(newpage) != 0)
+	if (fuse_check_folio(newfolio) != 0)
 		goto out_fallback_unlock;
 
 	/*
 	 * This is a new and locked page, it shouldn't be mapped or
 	 * have any special flags on it
 	 */
-	if (WARN_ON(page_mapped(oldpage)))
+	if (WARN_ON(folio_mapped(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(page_has_private(oldpage)))
+	if (WARN_ON(folio_has_private(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
+	if (WARN_ON(folio_test_dirty(oldfolio) ||
+				folio_test_writeback(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(PageMlocked(oldpage)))
+	if (WARN_ON(folio_test_mlocked(oldfolio)))
 		goto out_fallback_unlock;
 
-	replace_page_cache_folio(page_folio(oldpage), page_folio(newpage));
+	replace_page_cache_folio(oldfolio, newfolio);
 
-	get_page(newpage);
+	folio_get(newfolio);
 
 	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
-		lru_cache_add(newpage);
+		folio_add_lru(newfolio);
 
 	/*
 	 * Release while we have extra ref on stolen page.  Otherwise
@@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 	if (test_bit(FR_ABORTED, &cs->req->flags))
 		err = -ENOENT;
 	else
-		*pagep = newpage;
+		*pagep = &newfolio->page;
 	spin_unlock(&cs->req->waitq.lock);
 
 	if (err) {
-		unlock_page(newpage);
-		put_page(newpage);
+		folio_unlock(newfolio);
+		folio_put(newfolio);
 		goto out_put_old;
 	}
 
-	unlock_page(oldpage);
+	folio_unlock(oldfolio);
 	/* Drop ref for ap->pages[] array */
-	put_page(oldpage);
+	folio_put(oldfolio);
 	cs->len = 0;
 
 	err = 0;
 out_put_old:
 	/* Drop ref obtained in this function */
-	put_page(oldpage);
+	folio_put(oldfolio);
 	return err;
 
 out_fallback_unlock:
-	unlock_page(newpage);
+	folio_unlock(newfolio);
 out_fallback:
 	cs->pg = buf->page;
 	cs->offset = buf->offset;
Signature · 1 line
2.38.1
maintainer
#4 lore raw

Re: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios

Quoted 3 lines · Vishal Moola (Oracle)

Converts the function to try to move folios instead of pages. Also converts fuse_check_page() to fuse_get_folio() since this is its only caller. This change removes 15 calls to compound_head().

This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?

Quoted 138 lines
---
 fs/fuse/dev.c | 55 ++++++++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)
fs/fuse/dev.c +28 -27
index 26817a2db463..204c332cd343 100644
@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
 	return ncpy;
 }
 
-static int fuse_check_page(struct page *page)
+static int fuse_check_folio(struct folio *folio)
 {
-	if (page_mapcount(page) ||
-	    page->mapping != NULL ||
-	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
+	if (folio_mapped(folio) ||
+	    folio->mapping != NULL ||
+	    (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &
 	     ~(1 << PG_locked |
 	       1 << PG_referenced |
 	       1 << PG_uptodate |
@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page)
 	       1 << PG_reclaim |
 	       1 << PG_waiters |
 	       LRU_GEN_MASK | LRU_REFS_MASK))) {
-		dump_page(page, "fuse: trying to steal weird page");
+		dump_page(&folio->page, "fuse: trying to steal weird page");
 		return 1;
 	}
 	return 0;
@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)
 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 {
 	int err;
-	struct page *oldpage = *pagep;
-	struct page *newpage;
+	struct folio *oldfolio = page_folio(*pagep);
+	struct folio *newfolio;
 	struct pipe_buffer *buf = cs->pipebufs;
 
-	get_page(oldpage);
+	folio_get(oldfolio);
 	err = unlock_request(cs->req);
 	if (err)
 		goto out_put_old;
@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 	if (!pipe_buf_try_steal(cs->pipe, buf))
 		goto out_fallback;
 
-	newpage = buf->page;
+	newfolio = page_folio(buf->page);
 
-	if (!PageUptodate(newpage))
-		SetPageUptodate(newpage);
+	if (!folio_test_uptodate(newfolio))
+		folio_mark_uptodate(newfolio);
 
-	ClearPageMappedToDisk(newpage);
+	folio_clear_mappedtodisk(newfolio);
 
-	if (fuse_check_page(newpage) != 0)
+	if (fuse_check_folio(newfolio) != 0)
 		goto out_fallback_unlock;
 
 	/*
 	 * This is a new and locked page, it shouldn't be mapped or
 	 * have any special flags on it
 	 */
-	if (WARN_ON(page_mapped(oldpage)))
+	if (WARN_ON(folio_mapped(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(page_has_private(oldpage)))
+	if (WARN_ON(folio_has_private(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
+	if (WARN_ON(folio_test_dirty(oldfolio) ||
+				folio_test_writeback(oldfolio)))
 		goto out_fallback_unlock;
-	if (WARN_ON(PageMlocked(oldpage)))
+	if (WARN_ON(folio_test_mlocked(oldfolio)))
 		goto out_fallback_unlock;
 
-	replace_page_cache_folio(page_folio(oldpage), page_folio(newpage));
+	replace_page_cache_folio(oldfolio, newfolio);
 
-	get_page(newpage);
+	folio_get(newfolio);
 
 	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
-		lru_cache_add(newpage);
+		folio_add_lru(newfolio);
 
 	/*
 	 * Release while we have extra ref on stolen page.  Otherwise
@@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 	if (test_bit(FR_ABORTED, &cs->req->flags))
 		err = -ENOENT;
 	else
-		*pagep = newpage;
+		*pagep = &newfolio->page;
 	spin_unlock(&cs->req->waitq.lock);
 
 	if (err) {
-		unlock_page(newpage);
-		put_page(newpage);
+		folio_unlock(newfolio);
+		folio_put(newfolio);
 		goto out_put_old;
 	}
 
-	unlock_page(oldpage);
+	folio_unlock(oldfolio);
 	/* Drop ref for ap->pages[] array */
-	put_page(oldpage);
+	folio_put(oldfolio);
 	cs->len = 0;
 
 	err = 0;
 out_put_old:
 	/* Drop ref obtained in this function */
-	put_page(oldpage);
+	folio_put(oldfolio);
 	return err;
 
 out_fallback_unlock:
-	unlock_page(newpage);
+	folio_unlock(newfolio);
 out_fallback:
 	cs->pg = buf->page;
 	cs->offset = buf->offset;
Signature · 1 line
2.38.1
#5 lore raw

Re: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios

Quoted 11 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)

Converts the function to try to move folios instead of pages. Also converts fuse_check_page() to fuse_get_folio() since this is its only caller. This change removes 15 calls to compound_head().

This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?

Miklos, could you please look over this patch?

Quoted 138 lines
Quoted 138 lines
---
 fs/fuse/dev.c | 55 ++++++++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)
fs/fuse/dev.c +28 -28
index 26817a2db463..204c332cd343 100644
@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
      return ncpy;
 }
 
-static int fuse_check_page(struct page *page)
+static int fuse_check_folio(struct folio *folio)
 {
-     if (page_mapcount(page) ||
-         page->mapping != NULL ||
-         (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
+     if (folio_mapped(folio) ||
+         folio->mapping != NULL ||
+         (folio->flags & PAGE_FLAGS_CHECK_AT_PREP &
           ~(1 << PG_locked |
             1 << PG_referenced |
             1 << PG_uptodate |
@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page)
             1 << PG_reclaim |
             1 << PG_waiters |
             LRU_GEN_MASK | LRU_REFS_MASK))) {
-             dump_page(page, "fuse: trying to steal weird page");
+             dump_page(&folio->page, "fuse: trying to steal weird page");
              return 1;
      }
      return 0;
@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)
 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
 {
      int err;
-     struct page *oldpage = *pagep;
-     struct page *newpage;
+     struct folio *oldfolio = page_folio(*pagep);
+     struct folio *newfolio;
      struct pipe_buffer *buf = cs->pipebufs;
 
-     get_page(oldpage);
+     folio_get(oldfolio);
      err = unlock_request(cs->req);
      if (err)
              goto out_put_old;
@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
      if (!pipe_buf_try_steal(cs->pipe, buf))
              goto out_fallback;
 
-     newpage = buf->page;
+     newfolio = page_folio(buf->page);
 
-     if (!PageUptodate(newpage))
-             SetPageUptodate(newpage);
+     if (!folio_test_uptodate(newfolio))
+             folio_mark_uptodate(newfolio);
 
-     ClearPageMappedToDisk(newpage);
+     folio_clear_mappedtodisk(newfolio);
 
-     if (fuse_check_page(newpage) != 0)
+     if (fuse_check_folio(newfolio) != 0)
              goto out_fallback_unlock;
 
      /*
       * This is a new and locked page, it shouldn't be mapped or
       * have any special flags on it
       */
-     if (WARN_ON(page_mapped(oldpage)))
+     if (WARN_ON(folio_mapped(oldfolio)))
              goto out_fallback_unlock;
-     if (WARN_ON(page_has_private(oldpage)))
+     if (WARN_ON(folio_has_private(oldfolio)))
              goto out_fallback_unlock;
-     if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
+     if (WARN_ON(folio_test_dirty(oldfolio) ||
+                             folio_test_writeback(oldfolio)))
              goto out_fallback_unlock;
-     if (WARN_ON(PageMlocked(oldpage)))
+     if (WARN_ON(folio_test_mlocked(oldfolio)))
              goto out_fallback_unlock;
 
-     replace_page_cache_folio(page_folio(oldpage), page_folio(newpage));
+     replace_page_cache_folio(oldfolio, newfolio);
 
-     get_page(newpage);
+     folio_get(newfolio);
 
      if (!(buf->flags & PIPE_BUF_FLAG_LRU))
-             lru_cache_add(newpage);
+             folio_add_lru(newfolio);
 
      /*
       * Release while we have extra ref on stolen page.  Otherwise
@@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
      if (test_bit(FR_ABORTED, &cs->req->flags))
              err = -ENOENT;
      else
-             *pagep = newpage;
+             *pagep = &newfolio->page;
      spin_unlock(&cs->req->waitq.lock);
 
      if (err) {
-             unlock_page(newpage);
-             put_page(newpage);
+             folio_unlock(newfolio);
+             folio_put(newfolio);
              goto out_put_old;
      }
 
-     unlock_page(oldpage);
+     folio_unlock(oldfolio);
      /* Drop ref for ap->pages[] array */
-     put_page(oldpage);
+     folio_put(oldfolio);
      cs->len = 0;
 
      err = 0;
 out_put_old:
      /* Drop ref obtained in this function */
-     put_page(oldpage);
+     folio_put(oldfolio);
      return err;
 
 out_fallback_unlock:
-     unlock_page(newpage);
+     folio_unlock(newfolio);
 out_fallback:
      cs->pg = buf->page;
      cs->offset = buf->offset;
--

2.38.1

maintainer Acked-by
#6 lore raw

Re: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios

Quoted 8 lines · Vishal Moola
Quoted 6 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)

Converts the function to try to move folios instead of pages. Also converts fuse_check_page() to fuse_get_folio() since this is its only caller. This change removes 15 calls to compound_head().

Quoted 7 lines
Quoted 5 lines

This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?

Miklos, could you please look over this patch?

I think this should take care of the large folio case in fuse_try_move_page():

    if (cs->len != PAGE_SIZE)
        goto out_fallback;

The patch looks okay.

Thanks,
Miklos

#7 lore raw

PATCH 3/5 userfualtfd: Replace lru_cache functions with folio_add functions

In reply to Vishal Moola (Oracle)

Replaces lru_cache_add() and lru_cache_add_inactive_or_unevictable() with folio_add_lru() and folio_add_lru_vma(). This is in preparation for the removal of lru_cache_add().

---
 mm/userfaultfd.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/userfaultfd.c +4 -2
index e24e8a47ce8a..2560973b00d8 100644
@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
 	bool page_in_cache = page->mapping;
 	spinlock_t *ptl;
+	struct folio *folio;
 	struct inode *inode;
 	pgoff_t offset, max_off;
 
@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	if (!pte_none_mostly(*dst_pte))
 		goto out_unlock;
 
+	folio = page_folio(page);
 	if (page_in_cache) {
 		/* Usually, cache pages are already added to LRU */
 		if (newly_allocated)
-			lru_cache_add(page);
+			folio_add_lru(folio);
 		page_add_file_rmap(page, dst_vma, false);
 	} else {
 		page_add_new_anon_rmap(page, dst_vma, dst_addr);
-		lru_cache_add_inactive_or_unevictable(page, dst_vma);
+		folio_add_lru_vma(folio, dst_vma);
 	}
 
 	/*
Signature · 1 line
2.38.1
maintainer
#8 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 3 lines · Vishal Moola (Oracle)

Replaces lru_cache_add() and lru_cache_add_inactive_or_unevictable() with folio_add_lru() and folio_add_lru_vma(). This is in preparation for the removal of lru_cache_add().

Ummmmm. Reviewing this patch reveals a bug (not introduced by your patch). Look:

mfill_atomic_install_pte:

        bool page_in_cache = page->mapping;

mcontinue_atomic_pte:

        ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC);
...
        page = folio_file_page(folio, pgoff);
        ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
                                       page, false, wp_copy);

That says pretty plainly that mfill_atomic_install_pte() can be passed a tail page from shmem, and if it is ...

        if (page_in_cache) {
...
        } else {
                page_add_new_anon_rmap(page, dst_vma, dst_addr);
                lru_cache_add_inactive_or_unevictable(page, dst_vma);
        }

it'll get put on the rmap as an anon page!

Quoted 39 lines
---
 mm/userfaultfd.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/userfaultfd.c +4 -2
index e24e8a47ce8a..2560973b00d8 100644
@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
 	bool page_in_cache = page->mapping;
 	spinlock_t *ptl;
+	struct folio *folio;
 	struct inode *inode;
 	pgoff_t offset, max_off;
 
@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	if (!pte_none_mostly(*dst_pte))
 		goto out_unlock;
 
+	folio = page_folio(page);
 	if (page_in_cache) {
 		/* Usually, cache pages are already added to LRU */
 		if (newly_allocated)
-			lru_cache_add(page);
+			folio_add_lru(folio);
 		page_add_file_rmap(page, dst_vma, false);
 	} else {
 		page_add_new_anon_rmap(page, dst_vma, dst_addr);
-		lru_cache_add_inactive_or_unevictable(page, dst_vma);
+		folio_add_lru_vma(folio, dst_vma);
 	}
 
 	/*
Signature · 1 line
2.38.1
reviewer
#9 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 21 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)

Replaces lru_cache_add() and lru_cache_add_inactive_or_unevictable() with folio_add_lru() and folio_add_lru_vma(). This is in preparation for the removal of lru_cache_add().

Ummmmm. Reviewing this patch reveals a bug (not introduced by your patch). Look:

mfill_atomic_install_pte:

        bool page_in_cache = page->mapping;

mcontinue_atomic_pte:

        ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC);
...
        page = folio_file_page(folio, pgoff);
        ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
                                       page, false, wp_copy);

That says pretty plainly that mfill_atomic_install_pte() can be passed a tail page from shmem, and if it is ...

Quoted 8 lines
        if (page_in_cache) {
...
        } else {
                page_add_new_anon_rmap(page, dst_vma, dst_addr);
                lru_cache_add_inactive_or_unevictable(page, dst_vma);
        }

it'll get put on the rmap as an anon page!

Hmm yeah.. thanks Matthew!

Does the patch attached look reasonable to you?

Copying Axel too.

Quoted 41 lines
Quoted 40 lines
---
 mm/userfaultfd.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
mm/userfaultfd.c +4 -2
index e24e8a47ce8a..2560973b00d8 100644
@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
 	bool page_in_cache = page->mapping;
 	spinlock_t *ptl;
+	struct folio *folio;
 	struct inode *inode;
 	pgoff_t offset, max_off;
 
@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	if (!pte_none_mostly(*dst_pte))
 		goto out_unlock;
 
+	folio = page_folio(page);
 	if (page_in_cache) {
 		/* Usually, cache pages are already added to LRU */
 		if (newly_allocated)
-			lru_cache_add(page);
+			folio_add_lru(folio);
 		page_add_file_rmap(page, dst_vma, false);
 	} else {
 		page_add_new_anon_rmap(page, dst_vma, dst_addr);
-		lru_cache_add_inactive_or_unevictable(page, dst_vma);
+		folio_add_lru_vma(folio, dst_vma);
 	}
 
 	/*
Signature · 1 line
2.38.1
Signature · 1 line
Peter Xu

From 4eea0908b4890745bedd931283c1af91f509d039 Mon Sep 17 00:00:00 2001
From: Peter Xu <[email protected]>
Date: Wed, 2 Nov 2022 14:41:52 -0400
Subject: [PATCH] mm/shmem: Use page_mapping() to detect page cache for uffd
continue
Content-type: text/plain

mfill_atomic_install_pte() checks page->mapping to detect whether one page is used in the page cache. However as pointed out by Matthew, the page can logically be a tail page rather than always the head in the case of uffd minor mode with UFFDIO_CONTINUE. It means we could wrongly install one pte with shmem thp tail page assuming it's an anonymous page.

It's not that clear even for anonymous page, since normally anonymous pages also have page->mapping being setup with the anon vma. It's safe here only because the only such caller to mfill_atomic_install_pte() is always passing in a newly allocated page (mcopy_atomic_pte()), whose page->mapping is not yet setup. However that's not extremely obvious either.

For either of above, use page_mapping() instead.

And this should be stable material.

---
 mm/userfaultfd.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
mm/userfaultfd.c +1 -1
index 3d0fef3980b3..650ab6cfd5f4 100644
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	pte_t _dst_pte, *dst_pte;
 	bool writable = dst_vma->vm_flags & VM_WRITE;
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
-	bool page_in_cache = page->mapping;
+	bool page_in_cache = page_mapping(page);
 	spinlock_t *ptl;
 	struct inode *inode;
 	pgoff_t offset, max_off;
Signature · 1 line
2.37.3
maintainer
#10 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 1 line · Peter Xu

Does the patch attached look reasonable to you?

Mmm, no. If the page is in the swap cache, this will be "true".

mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67

Quoted 10 lines
mm/userfaultfd.c +1 -1
index 3d0fef3980b3..650ab6cfd5f4 100644
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	pte_t _dst_pte, *dst_pte;
 	bool writable = dst_vma->vm_flags & VM_WRITE;
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
-	bool page_in_cache = page->mapping;
+	bool page_in_cache = page_mapping(page);

We could do:

	struct page *head = compound_head(page);
	bool page_in_cache = head->mapping && !PageMappingFlags(head);
reviewer
#11 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu

Does the patch attached look reasonable to you?

Mmm, no. If the page is in the swap cache, this will be "true".

It will not happen in practise, right?

I mean, shmem_get_folio() should have done the swap-in, and we should have the page lock held at the meantime.

For anon, mcopy_atomic_pte() is the only user and it's passing in a newly allocated page here.

Quoted 14 lines

mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67

Quoted 11 lines
mm/userfaultfd.c +1 -1
index 3d0fef3980b3..650ab6cfd5f4 100644
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
 	pte_t _dst_pte, *dst_pte;
 	bool writable = dst_vma->vm_flags & VM_WRITE;
 	bool vm_shared = dst_vma->vm_flags & VM_SHARED;
-	bool page_in_cache = page->mapping;
+	bool page_in_cache = page_mapping(page);

We could do:

Quoted 2 lines
	struct page *head = compound_head(page);
	bool page_in_cache = head->mapping && !PageMappingFlags(head);

Sounds good to me, but it just gets a bit complicated.

If page_mapping() doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.

Signature · 1 line
Peter Xu
#12 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 29 lines · Peter Xu
Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu

Does the patch attached look reasonable to you?

Mmm, no. If the page is in the swap cache, this will be "true".

It will not happen in practise, right?

I mean, shmem_get_folio() should have done the swap-in, and we should have the page lock held at the meantime.

For anon, mcopy_atomic_pte() is the only user and it's passing in a newly allocated page here.

Quoted 14 lines

mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67

Quoted 11 lines
mm/userfaultfd.c +1 -1
index 3d0fef3980b3..650ab6cfd5f4 100644
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
    pte_t _dst_pte, *dst_pte;
    bool writable = dst_vma->vm_flags & VM_WRITE;
    bool vm_shared = dst_vma->vm_flags & VM_SHARED;
-   bool page_in_cache = page->mapping;
+   bool page_in_cache = page_mapping(page);

We could do:

Quoted 7 lines
Quoted 2 lines
      struct page *head = compound_head(page);
      bool page_in_cache = head->mapping && !PageMappingFlags(head);

Sounds good to me, but it just gets a bit complicated.

If page_mapping() doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.

For what it's worth, I think I like Matthew's version better than the original patch. This is because, although page_mapping() looks simpler here, looking into the definition of page_mapping() I feel it's handling several cases, not all of which are relevant here (or, as Matthew points out, would actually be wrong if it were possible to reach those cases here).

It's not clear to me what is meant by "pass that over from callers"? Do you mean, have callers pass in true/false for page_in_cache directly?

That could work, but I still think I prefer Matthew's version slightly better, if only because this function already takes a lot of arguments.

Quoted 4 lines

--
Peter Xu

reviewer
#13 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

Quoted 45 lines · Axel Rasmussen
Quoted 29 lines · Peter Xu
Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu

Does the patch attached look reasonable to you?

Mmm, no. If the page is in the swap cache, this will be "true".

It will not happen in practise, right?

I mean, shmem_get_folio() should have done the swap-in, and we should have the page lock held at the meantime.

For anon, mcopy_atomic_pte() is the only user and it's passing in a newly allocated page here.

Quoted 14 lines

mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67

Quoted 11 lines
mm/userfaultfd.c +1 -1
index 3d0fef3980b3..650ab6cfd5f4 100644
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
    pte_t _dst_pte, *dst_pte;
    bool writable = dst_vma->vm_flags & VM_WRITE;
    bool vm_shared = dst_vma->vm_flags & VM_SHARED;
-   bool page_in_cache = page->mapping;
+   bool page_in_cache = page_mapping(page);

We could do:

Quoted 7 lines
Quoted 2 lines
      struct page *head = compound_head(page);
      bool page_in_cache = head->mapping && !PageMappingFlags(head);

Sounds good to me, but it just gets a bit complicated.

If page_mapping() doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.

For what it's worth, I think I like Matthew's version better than the original patch. This is because, although page_mapping() looks simpler here, looking into the definition of page_mapping() I feel it's handling several cases, not all of which are relevant here (or, as Matthew points out, would actually be wrong if it were possible to reach those cases here).

Quoted 3 lines

It's not clear to me what is meant by "pass that over from callers"? Do you mean, have callers pass in true/false for page_in_cache directly?

Yes.

Quoted 4 lines

That could work, but I still think I prefer Matthew's version slightly better, if only because this function already takes a lot of arguments.

IMHO that's not an issue, we can merge them into flags, cleaning things alongside.

The simplest so far is still just to use page_mapping() to me, but no strong opinion here.

If to go with Matthew's patch, it'll be great if we can add a comment showing what we're doing (something like "Unwrapped page_mapping() but avoid looking into swap cache" would be good enough to me).

Thanks,

Signature · 1 line
Peter Xu
maintainer
#14 lore raw

Re: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions

In reply to Peter Xu
Quoted 12 lines · On Wed, 2 Nov 2022 15:02:35 -0400 Peter Xu <[email protected]> wrote:

mfill_atomic_install_pte() checks page->mapping to detect whether one page is used in the page cache. However as pointed out by Matthew, the page can logically be a tail page rather than always the head in the case of uffd minor mode with UFFDIO_CONTINUE. It means we could wrongly install one pte with shmem thp tail page assuming it's an anonymous page.

It's not that clear even for anonymous page, since normally anonymous pages also have page->mapping being setup with the anon vma. It's safe here only because the only such caller to mfill_atomic_install_pte() is always passing in a newly allocated page (mcopy_atomic_pte()), whose page->mapping is not yet setup. However that's not extremely obvious either.

Quoted 3 lines

For either of above, use page_mapping() instead.

And this should be stable material.

I added

Fixes 153132571f02 ("userfaultfd/shmem: support UFFDIO_CONTINUE for shmem")
#15 lore raw

PATCH 4/5 khugepage: Replace lru_cache_add() with folio_add_lru()

In reply to Vishal Moola (Oracle)

Replaces some calls with their folio equivalents. This is in preparation for the removal of lru_cache_add(). This replaces 3 calls to compound_head() with 1.

---
 mm/khugepaged.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)
mm/khugepaged.c +7 -4
index 4734315f7940..e432d5279043 100644
@@ -1970,6 +1970,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,
 
 	if (result == SCAN_SUCCEED) {
 		struct page *page, *tmp;
+		struct folio *folio;
 
 		/*
 		 * Replacing old pages with new one has succeeded, now we
@@ -1997,11 +1998,13 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,
 			index++;
 		}
 
-		SetPageUptodate(hpage);
-		page_ref_add(hpage, HPAGE_PMD_NR - 1);
+		folio = page_folio(hpage);
+		folio_mark_uptodate(folio);
+		folio_ref_add(folio, HPAGE_PMD_NR - 1);
+
 		if (is_shmem)
-			set_page_dirty(hpage);
-		lru_cache_add(hpage);
+			folio_mark_dirty(folio);
+		folio_add_lru(folio);
 
 		/*
 		 * Remove pte page tables, so we can re-fault the page as huge.
Signature · 1 line
2.38.1
#17 lore raw

PATCH 5/5 folio-compat: Remove lru_cache_add()

In reply to Vishal Moola (Oracle)

There are no longer any callers of lru_cache_add(), so remove it. This saves 107 bytes of kernel text. Also cleanup some comments such that they reference the new folio_add_lru() instead.

---
 include/linux/swap.h | 1 -
 mm/folio-compat.c    | 6 ------
 mm/truncate.c        | 2 +-
 mm/workingset.c      | 2 +-
 4 files changed, 2 insertions(+), 9 deletions(-)
include/linux/swap.h +0 -1
index a18cf4b7c724..c92ccff9b962 100644
@@ -388,7 +388,6 @@ void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages);
 void lru_note_cost_folio(struct folio *);
 void folio_add_lru(struct folio *);
 void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
-void lru_cache_add(struct page *);
 void mark_page_accessed(struct page *);
 void folio_mark_accessed(struct folio *);
 
mm/folio-compat.c +0 -6
index e1e23b4947d7..efd65b7f48bb 100644
@@ -82,12 +82,6 @@ bool redirty_page_for_writepage(struct writeback_control *wbc,
 }
 EXPORT_SYMBOL(redirty_page_for_writepage);
 
-void lru_cache_add(struct page *page)
-{
-	folio_add_lru(page_folio(page));
-}
-EXPORT_SYMBOL(lru_cache_add);
-
 void lru_cache_add_inactive_or_unevictable(struct page *page,
 		struct vm_area_struct *vma)
 {
mm/truncate.c +1 -1
index c0be77e5c008..184fa17fce60 100644
@@ -573,7 +573,7 @@ EXPORT_SYMBOL(invalidate_mapping_pages);
  * refcount.  We do this because invalidate_inode_pages2() needs stronger
  * invalidation guarantees, and cannot afford to leave pages behind because
  * shrink_page_list() has a temp ref on them, or because they're transiently
- * sitting in the lru_cache_add() pagevecs.
+ * sitting in the folio_add_lru() pagevecs.
  */
 static int invalidate_complete_folio2(struct address_space *mapping,
 					struct folio *folio)
mm/workingset.c +1 -1
index ae7e984b23c6..25844171b72d 100644
@@ -492,7 +492,7 @@ void workingset_refault(struct folio *folio, void *shadow)
 	/* Folio was active prior to eviction */
 	if (workingset) {
 		folio_set_workingset(folio);
-		/* XXX: Move to lru_cache_add() when it supports new vs putback */
+		/* XXX: Move to folio_add_lru() when it supports new vs putback */
 		lru_note_cost_folio(folio);
 		mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
 	}
Signature · 1 line
2.38.1
#19 lore raw
In reply to Vishal Moola (Oracle)
Quoted 4 lines · Vishal Moola (Oracle)

This patchset replaces all calls of lru_cache_add() with the folio
equivalent: folio_add_lru(). This is allows us to get rid of the wrapper The series passes xfstests and the userfaultfd selftests.

All of these patches have been reviewed. Andrew, is there anything you'd like to see changed, or will you take these?