Removing the lru_cache_add() wrapper
Subsystems (10)
cover Removing the lru_cache_add() wrapper
This patchset replaces all calls of lru_cache_add() with the folio
equivalent: folio_add_lru(). This is allows us to get rid of the wrapper The series passes xfstests and the userfaultfd selftests.
I haven't been able to hit the fuse_try_move_page() function, so I haven't been able to test those changes. Since replace_page_cache_page() is only called by fuse_try_move_page() I haven't been able to test that as well. Testing and review of both would be beneficial.
Vishal Moola (Oracle) (5):
filemap: Convert replace_page_cache_page() to
replace_page_cache_folio()
fuse: Convert fuse_try_move_page() to use folios
userfualtfd: Replace lru_cache functions with folio_add functions
khugepage: Replace lru_cache_add() with folio_add_lru()
folio-compat: Remove lru_cache_add()
fs/fuse/dev.c | 55 +++++++++++++++++++++-------------------- include/linux/pagemap.h | 2 +- include/linux/swap.h | 1 - mm/filemap.c | 52 +++++++++++++++++++------------------- mm/folio-compat.c | 6 ----- mm/khugepaged.c | 11 ++++++--- mm/truncate.c | 2 +- mm/userfaultfd.c | 6 +++-- mm/workingset.c | 2 +- 9 files changed, 67 insertions(+), 70 deletions(-)
Signature · 1 line
2.38.1
PATCH 1/5 filemap: Convert replace_page_cache_page() to replace_page_cache_folio()
Eliminates 7 calls to compound_head().
--- fs/fuse/dev.c | 2 +- include/linux/pagemap.h | 2 +- mm/filemap.c | 52 ++++++++++++++++++++--------------------- 3 files changed, 27 insertions(+), 29 deletions(-)
fs/fuse/dev.c +1 -1
@@ -837,7 +837,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (WARN_ON (PageMlocked (oldpage))) goto out_fallback_unlock; -replace_page_cache_page (oldpage, newpage); +replace_page_cache_folio (page_folio (oldpage),page_folio (newpage));get_page (newpage);
include/linux/pagemap.h +1 -1
@@ -1104,7 +1104,7 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio,void filemap_remove_folio (struct folio *folio);void delete_from_page_cache (struct page *page);void __filemap_remove_folio (struct folio *folio,void *shadow); -void replace_page_cache_page (struct page *old,struct page *new); +void replace_page_cache_folio (struct folio *old,struct folio *new);void delete_from_page_cache_batch (struct address_space *mapping,struct folio_batch *fbatch);int try_to_release_page (struct page *page,gfp_t gfp);
mm/filemap.c +25 -27
81 lines not on this page · raw message
Signature · 1 line
2.38.1
Re: [PATCH 1/5] filemap: Convert replace_page_cache_page() to replace_page_cache_folio()
Quoted 3 lines · Vishal Moola (Oracle)
Eliminates 7 calls to
compound_head().Signed-off-by Vishal Moola (Oracle) <[email protected]>
PATCH 2/5 fuse: Convert fuse_try_move_page() to use folios
In reply to Vishal Moola (Oracle)Converts the function to try to move folios instead of pages. Also converts fuse_check_page() to fuse_get_folio() since this is its only caller. This change removes 15 calls to compound_head().
--- fs/fuse/dev.c | 55 ++++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 27 deletions(-)
fs/fuse/dev.c +28 -27
@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)return ncpy; } -static int fuse_check_page (struct page *page) +static int fuse_check_folio (struct folio *folio) { - if (page_mapcount(page) || - page->mapping != NULL || - (page->flags & PAGE_FLAGS_CHECK_AT_PREP & + if (folio_mapped(folio) || + folio->mapping != NULL || + (folio->flags & PAGE_FLAGS_CHECK_AT_PREP & ~(1 << PG_locked | 1 << PG_referenced | 1 << PG_uptodate |
@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page) 1 << PG_reclaim | 1 << PG_waiters | LRU_GEN_MASK | LRU_REFS_MASK))) { -dump_page (page,"fuse: trying to steal weird page" ); +dump_page (&folio->page,"fuse: trying to steal weird page" );return 1 ; }return 0 ;
@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)static int fuse_try_move_page (struct fuse_copy_state *cs,struct page **pagep) {int err; -struct page *oldpage = *pagep; -struct page *newpage; +struct folio *oldfolio =page_folio (*pagep); +struct folio *newfolio;struct pipe_buffer *buf = cs->pipebufs; -get_page (oldpage); +folio_get (oldfolio); err =unlock_request (cs->req);if (err) goto out_put_old;
@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (!pipe_buf_try_steal (cs->pipe, buf)) goto out_fallback; - newpage = buf->page; + newfolio =page_folio (buf->page); -if (!PageUptodate (newpage)) -SetPageUptodate (newpage); +if (!folio_test_uptodate (newfolio)) +folio_mark_uptodate (newfolio); -ClearPageMappedToDisk (newpage); +folio_clear_mappedtodisk (newfolio); -if (fuse_check_page (newpage) !=0 ) +if (fuse_check_folio (newfolio) !=0 ) goto out_fallback_unlock;/* * This is a new and locked page, it shouldn't be mapped or * have any special flags on it */ -if (WARN_ON (page_mapped (oldpage))) +if (WARN_ON (folio_mapped (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (page_has_private (oldpage))) +if (WARN_ON (folio_has_private (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageDirty (oldpage) ||PageWriteback (oldpage))) +if (WARN_ON (folio_test_dirty (oldfolio) || +folio_test_writeback (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageMlocked (oldpage))) +if (WARN_ON (folio_test_mlocked (oldfolio))) goto out_fallback_unlock; -replace_page_cache_folio (page_folio (oldpage),page_folio (newpage)); +replace_page_cache_folio (oldfolio, newfolio); -get_page (newpage); +folio_get (newfolio);if (!(buf->flags &PIPE_BUF_FLAG_LRU )) -lru_cache_add (newpage); +folio_add_lru (newfolio);/* * Release while we have extra ref on stolen page. Otherwise
@@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (test_bit (FR_ABORTED , &cs->req->flags)) err = -ENOENT ;else - *pagep = newpage; + *pagep = &newfolio->page;spin_unlock (&cs->req->waitq.lock);if (err) { -unlock_page (newpage); -put_page (newpage); +folio_unlock (newfolio); +folio_put (newfolio); goto out_put_old; } -unlock_page (oldpage); +folio_unlock (oldfolio);/* Drop ref for ap->pages[] array */ -put_page (oldpage); +folio_put (oldfolio); cs->len =0 ; err =0 ; out_put_old:/* Drop ref obtained in this function */ -put_page (oldpage); +folio_put (oldfolio);return err; out_fallback_unlock: -unlock_page (newpage); +folio_unlock (newfolio); out_fallback: cs->pg = buf->page; cs->offset = buf->offset;
Signature · 1 line
2.38.1
Matthew Wilcox
maintainerRe: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios
Quoted 3 lines · Vishal Moola (Oracle)
Converts the function to try to move folios instead of pages. Also converts
fuse_check_page()tofuse_get_folio()since this is its only caller. This change removes 15 calls tocompound_head().
This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?
Quoted 138 lines
Signed-off-by Vishal Moola (Oracle) <[email protected]>--- fs/fuse/dev.c | 55 ++++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 27 deletions(-)fs/fuse/dev.c +28
-27@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)return ncpy; } -static int fuse_check_page (struct page *page) +static int fuse_check_folio (struct folio *folio) { - if (page_mapcount(page) || - page->mapping != NULL || - (page->flags & PAGE_FLAGS_CHECK_AT_PREP & + if (folio_mapped(folio) || + folio->mapping != NULL || + (folio->flags & PAGE_FLAGS_CHECK_AT_PREP & ~(1 << PG_locked | 1 << PG_referenced | 1 << PG_uptodate |@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page) 1 << PG_reclaim | 1 << PG_waiters | LRU_GEN_MASK | LRU_REFS_MASK))) { -dump_page (page,"fuse: trying to steal weird page" ); +dump_page (&folio->page,"fuse: trying to steal weird page" );return 1 ; }return 0 ;@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)static int fuse_try_move_page (struct fuse_copy_state *cs,struct page **pagep) {int err; -struct page *oldpage = *pagep; -struct page *newpage; +struct folio *oldfolio =page_folio (*pagep); +struct folio *newfolio;struct pipe_buffer *buf = cs->pipebufs; -get_page (oldpage); +folio_get (oldfolio); err =unlock_request (cs->req);if (err) goto out_put_old;@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (!pipe_buf_try_steal (cs->pipe, buf)) goto out_fallback; - newpage = buf->page; + newfolio =page_folio (buf->page); -if (!PageUptodate (newpage)) -SetPageUptodate (newpage); +if (!folio_test_uptodate (newfolio)) +folio_mark_uptodate (newfolio); -ClearPageMappedToDisk (newpage); +folio_clear_mappedtodisk (newfolio); -if (fuse_check_page (newpage) !=0 ) +if (fuse_check_folio (newfolio) !=0 ) goto out_fallback_unlock;/* * This is a new and locked page, it shouldn't be mapped or * have any special flags on it */ -if (WARN_ON (page_mapped (oldpage))) +if (WARN_ON (folio_mapped (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (page_has_private (oldpage))) +if (WARN_ON (folio_has_private (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageDirty (oldpage) ||PageWriteback (oldpage))) +if (WARN_ON (folio_test_dirty (oldfolio) || +folio_test_writeback (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageMlocked (oldpage))) +if (WARN_ON (folio_test_mlocked (oldfolio))) goto out_fallback_unlock; -replace_page_cache_folio (page_folio (oldpage),page_folio (newpage)); +replace_page_cache_folio (oldfolio, newfolio); -get_page (newpage); +folio_get (newfolio);if (!(buf->flags &PIPE_BUF_FLAG_LRU )) -lru_cache_add (newpage); +folio_add_lru (newfolio);/* * Release while we have extra ref on stolen page. Otherwise @@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (test_bit (FR_ABORTED , &cs->req->flags)) err = -ENOENT ;else - *pagep = newpage; + *pagep = &newfolio->page;spin_unlock (&cs->req->waitq.lock);if (err) { -unlock_page (newpage); -put_page (newpage); +folio_unlock (newfolio); +folio_put (newfolio); goto out_put_old; } -unlock_page (oldpage); +folio_unlock (oldfolio);/* Drop ref for ap->pages[] array */ -put_page (oldpage); +folio_put (oldfolio); cs->len =0 ; err =0 ; out_put_old:/* Drop ref obtained in this function */ -put_page (oldpage); +folio_put (oldfolio);return err; out_fallback_unlock: -unlock_page (newpage); +folio_unlock (newfolio); out_fallback: cs->pg = buf->page; cs->offset = buf->offset;Signature · 1 line
2.38.1
Re: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios
Quoted 11 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)
Converts the function to try to move folios instead of pages. Also converts
fuse_check_page()tofuse_get_folio()since this is its only caller. This change removes 15 calls tocompound_head().This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?
Miklos, could you please look over this patch?
Quoted 138 lines
Quoted 138 lines
Signed-off-by Vishal Moola (Oracle) <[email protected]>--- fs/fuse/dev.c | 55 ++++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 27 deletions(-)fs/fuse/dev.c +28
-28@@ -764,11 +764,11 @@ static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)return ncpy; } -static int fuse_check_page (struct page *page) +static int fuse_check_folio (struct folio *folio) { - if (page_mapcount(page) || - page->mapping != NULL || - (page->flags & PAGE_FLAGS_CHECK_AT_PREP & + if (folio_mapped(folio) || + folio->mapping != NULL || + (folio->flags & PAGE_FLAGS_CHECK_AT_PREP & ~(1 << PG_locked | 1 << PG_referenced | 1 << PG_uptodate |@@ -778,7 +778,7 @@ static int fuse_check_page(struct page *page) 1 << PG_reclaim | 1 << PG_waiters | LRU_GEN_MASK | LRU_REFS_MASK))) { -dump_page (page,"fuse: trying to steal weird page" ); +dump_page (&folio->page,"fuse: trying to steal weird page" );return 1 ; }return 0 ;@@ -787,11 +787,11 @@ static int fuse_check_page(struct page *page)static int fuse_try_move_page (struct fuse_copy_state *cs,struct page **pagep) {int err; -struct page *oldpage = *pagep; -struct page *newpage; +struct folio *oldfolio =page_folio (*pagep); +struct folio *newfolio;struct pipe_buffer *buf = cs->pipebufs; -get_page (oldpage); +folio_get (oldfolio); err =unlock_request (cs->req);if (err) goto out_put_old;@@ -814,35 +814,36 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (!pipe_buf_try_steal (cs->pipe, buf)) goto out_fallback; - newpage = buf->page; + newfolio =page_folio (buf->page); -if (!PageUptodate (newpage)) -SetPageUptodate (newpage); +if (!folio_test_uptodate (newfolio)) +folio_mark_uptodate (newfolio); -ClearPageMappedToDisk (newpage); +folio_clear_mappedtodisk (newfolio); -if (fuse_check_page (newpage) !=0 ) +if (fuse_check_folio (newfolio) !=0 ) goto out_fallback_unlock;/* * This is a new and locked page, it shouldn't be mapped or * have any special flags on it */ -if (WARN_ON (page_mapped (oldpage))) +if (WARN_ON (folio_mapped (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (page_has_private (oldpage))) +if (WARN_ON (folio_has_private (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageDirty (oldpage) ||PageWriteback (oldpage))) +if (WARN_ON (folio_test_dirty (oldfolio) || +folio_test_writeback (oldfolio))) goto out_fallback_unlock; -if (WARN_ON (PageMlocked (oldpage))) +if (WARN_ON (folio_test_mlocked (oldfolio))) goto out_fallback_unlock; -replace_page_cache_folio (page_folio (oldpage),page_folio (newpage)); +replace_page_cache_folio (oldfolio, newfolio); -get_page (newpage); +folio_get (newfolio);if (!(buf->flags &PIPE_BUF_FLAG_LRU )) -lru_cache_add (newpage); +folio_add_lru (newfolio);/* * Release while we have extra ref on stolen page. Otherwise @@ -855,28 +856,28 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)if (test_bit (FR_ABORTED , &cs->req->flags)) err = -ENOENT ;else - *pagep = newpage; + *pagep = &newfolio->page;spin_unlock (&cs->req->waitq.lock);if (err) { -unlock_page (newpage); -put_page (newpage); +folio_unlock (newfolio); +folio_put (newfolio); goto out_put_old; } -unlock_page (oldpage); +folio_unlock (oldfolio);/* Drop ref for ap->pages[] array */ -put_page (oldpage); +folio_put (oldfolio); cs->len =0 ; err =0 ; out_put_old:/* Drop ref obtained in this function */ -put_page (oldpage); +folio_put (oldfolio);return err; out_fallback_unlock: -unlock_page (newpage); +folio_unlock (newfolio); out_fallback: cs->pg = buf->page; cs->offset = buf->offset; --2.38.1
Re: [PATCH 2/5] fuse: Convert fuse_try_move_page() to use folios
Quoted 8 lines · Vishal Moola
Quoted 6 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)
Converts the function to try to move folios instead of pages. Also converts
fuse_check_page()tofuse_get_folio()since this is its only caller. This change removes 15 calls tocompound_head().
Quoted 7 lines
Quoted 5 lines
This all looks good. I wonder if we should't add an assertion that the page we're trying to steal is !large? It seems to me that there are assumptions in this part of fuse that it's only dealing with order-0 pages, and if someone gives it a page that's part of a large folio, it's going to be messy. Miklos, any thoughts?
Miklos, could you please look over this patch?
I think this should take care of the large folio case in fuse_try_move_page():
if (cs->len != PAGE_SIZE)
goto out_fallback;The patch looks okay.
Thanks,
Miklos
PATCH 3/5 userfualtfd: Replace lru_cache functions with folio_add functions
In reply to Vishal Moola (Oracle)Replaces lru_cache_add() and lru_cache_add_inactive_or_unevictable() with folio_add_lru() and folio_add_lru_vma(). This is in preparation for the removal of lru_cache_add().
--- mm/userfaultfd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
mm/userfaultfd.c +4 -2
@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,bool vm_shared = dst_vma->vm_flags &VM_SHARED ;bool page_in_cache = page->mapping;spinlock_t *ptl; +struct folio *folio;struct inode *inode;pgoff_t offset, max_off;
@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,if (!pte_none_mostly (*dst_pte)) goto out_unlock; + folio =page_folio (page);if (page_in_cache) {/* Usually, cache pages are already added to LRU */ if (newly_allocated) -lru_cache_add (page); +folio_add_lru (folio);page_add_file_rmap (page, dst_vma, false); }else {page_add_new_anon_rmap (page, dst_vma, dst_addr); -lru_cache_add_inactive_or_unevictable (page, dst_vma); +folio_add_lru_vma (folio, dst_vma); }/*
Signature · 1 line
2.38.1
Matthew Wilcox
maintainerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 3 lines · Vishal Moola (Oracle)
Replaces
lru_cache_add()andlru_cache_add_inactive_or_unevictable()withfolio_add_lru()andfolio_add_lru_vma(). This is in preparation for the removal oflru_cache_add().
Ummmmm. Reviewing this patch reveals a bug (not introduced by your patch). Look:
mfill_atomic_install_pte:
bool page_in_cache = page->mapping;
mcontinue_atomic_pte:
ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC);
...
page = folio_file_page(folio, pgoff);
ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
page, false, wp_copy);That says pretty plainly that mfill_atomic_install_pte() can be passed a tail page from shmem, and if it is ...
if (page_in_cache) {
...
} else {
page_add_new_anon_rmap(page, dst_vma, dst_addr);
lru_cache_add_inactive_or_unevictable(page, dst_vma);
}it'll get put on the rmap as an anon page!
Quoted 39 lines
Signed-off-by Vishal Moola (Oracle) <[email protected]>--- mm/userfaultfd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)mm/userfaultfd.c +4
-2@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,bool vm_shared = dst_vma->vm_flags &VM_SHARED ;bool page_in_cache = page->mapping;spinlock_t *ptl; +struct folio *folio;struct inode *inode;pgoff_t offset, max_off;@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,if (!pte_none_mostly (*dst_pte)) goto out_unlock; + folio =page_folio (page);if (page_in_cache) {/* Usually, cache pages are already added to LRU */ if (newly_allocated) -lru_cache_add (page); +folio_add_lru (folio);page_add_file_rmap (page, dst_vma, false); }else {page_add_new_anon_rmap (page, dst_vma, dst_addr); -lru_cache_add_inactive_or_unevictable (page, dst_vma); +folio_add_lru_vma (folio, dst_vma); }/* Signature · 1 line
2.38.1
Peter Xu
reviewerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 21 lines · Matthew Wilcox
Quoted 3 lines · Vishal Moola (Oracle)
Replaces
lru_cache_add()andlru_cache_add_inactive_or_unevictable()withfolio_add_lru()andfolio_add_lru_vma(). This is in preparation for the removal oflru_cache_add().Ummmmm. Reviewing this patch reveals a bug (not introduced by your patch). Look:
mfill_atomic_install_pte:bool page_in_cache = page->mapping;
mcontinue_atomic_pte:ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC); ... page = folio_file_page(folio, pgoff); ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, page, false, wp_copy);That says pretty plainly that
mfill_atomic_install_pte()can be passed a tail page from shmem, and if it is ...
Quoted 8 lines
if (page_in_cache) { ... } else { page_add_new_anon_rmap(page, dst_vma, dst_addr); lru_cache_add_inactive_or_unevictable(page, dst_vma); }it'll get put on the rmap as an anon page!
Hmm yeah.. thanks Matthew!
Does the patch attached look reasonable to you?
Copying Axel too.
Quoted 41 lines
Quoted 40 lines
Signed-off-by Vishal Moola (Oracle) <[email protected]>--- mm/userfaultfd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)mm/userfaultfd.c +4
-2@@ -66,6 +66,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,bool vm_shared = dst_vma->vm_flags &VM_SHARED ;bool page_in_cache = page->mapping;spinlock_t *ptl; +struct folio *folio;struct inode *inode;pgoff_t offset, max_off;@@ -113,14 +114,15 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,if (!pte_none_mostly (*dst_pte)) goto out_unlock; + folio =page_folio (page);if (page_in_cache) {/* Usually, cache pages are already added to LRU */ if (newly_allocated) -lru_cache_add (page); +folio_add_lru (folio);page_add_file_rmap (page, dst_vma, false); }else {page_add_new_anon_rmap (page, dst_vma, dst_addr); -lru_cache_add_inactive_or_unevictable (page, dst_vma); +folio_add_lru_vma (folio, dst_vma); }/* Signature · 1 line
2.38.1
Signature · 1 line
Peter Xu
From 4eea0908b4890745bedd931283c1af91f509d039 Mon Sep 17 00:00:00 2001
From: Peter Xu <[email protected]>
Date: Wed, 2 Nov 2022 14:41:52 -0400
Subject: [PATCH] mm/shmem: Use page_mapping() to detect page cache for uffd
continue
Content-type: text/plain
mfill_atomic_install_pte() checks page->mapping to detect whether one page is used in the page cache. However as pointed out by Matthew, the page can logically be a tail page rather than always the head in the case of uffd minor mode with UFFDIO_CONTINUE. It means we could wrongly install one pte with shmem thp tail page assuming it's an anonymous page.
It's not that clear even for anonymous page, since normally anonymous pages also have page->mapping being setup with the anon vma. It's safe here only because the only such caller to mfill_atomic_install_pte() is always passing in a newly allocated page (mcopy_atomic_pte()), whose page->mapping is not yet setup. However that's not extremely obvious either.
For either of above, use page_mapping() instead.
And this should be stable material.
--- mm/userfaultfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/userfaultfd.c +1 -1
@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,pte_t _dst_pte, *dst_pte;bool writable = dst_vma->vm_flags &VM_WRITE ;bool vm_shared = dst_vma->vm_flags &VM_SHARED ; -bool page_in_cache = page->mapping; +bool page_in_cache =page_mapping (page);spinlock_t *ptl;struct inode *inode;pgoff_t offset, max_off;
Signature · 1 line
2.37.3
Matthew Wilcox
maintainerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 1 line · Peter Xu
Does the patch attached look reasonable to you?
Mmm, no. If the page is in the swap cache, this will be "true".
mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67
Quoted 10 lines
mm/userfaultfd.c +1
-1@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,pte_t _dst_pte, *dst_pte;bool writable = dst_vma->vm_flags &VM_WRITE ;bool vm_shared = dst_vma->vm_flags &VM_SHARED ; -bool page_in_cache = page->mapping; +bool page_in_cache =page_mapping (page);
We could do:
struct page *head = compound_head(page); bool page_in_cache = head->mapping && !PageMappingFlags(head);
Peter Xu
reviewerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu
Does the patch attached look reasonable to you?
Mmm, no. If the page is in the swap cache, this will be "true".
It will not happen in practise, right?
I mean, shmem_get_folio() should have done the swap-in, and we should have the page lock held at the meantime.
For anon, mcopy_atomic_pte() is the only user and it's passing in a newly allocated page here.
Quoted 14 lines
mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67
Quoted 11 lines
mm/userfaultfd.c +1
-1@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,pte_t _dst_pte, *dst_pte;bool writable = dst_vma->vm_flags &VM_WRITE ;bool vm_shared = dst_vma->vm_flags &VM_SHARED ; -bool page_in_cache = page->mapping; +bool page_in_cache =page_mapping (page);We could do:
Quoted 2 lines
struct page *head = compound_head(page); bool page_in_cache = head->mapping && !PageMappingFlags(head);
Sounds good to me, but it just gets a bit complicated.
If page_mapping() doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.
Signature · 1 line
Peter Xu
Axel Rasmussen
reviewerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 29 lines · Peter Xu
Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu
Does the patch attached look reasonable to you?
Mmm, no. If the page is in the swap cache, this will be "true".
It will not happen in practise, right?
I mean,
shmem_get_folio()should have done the swap-in, and we should have the page lock held at the meantime.For anon,
mcopy_atomic_pte()is the only user and it's passing in a newly allocated page here.Quoted 14 lines
mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67
Quoted 11 lines
mm/userfaultfd.c +1
-1@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,pte_t _dst_pte, *dst_pte;bool writable = dst_vma->vm_flags &VM_WRITE ;bool vm_shared = dst_vma->vm_flags &VM_SHARED ; -bool page_in_cache = page->mapping; +bool page_in_cache =page_mapping (page);We could do:
Quoted 7 lines
Quoted 2 lines
struct page *head = compound_head(page); bool page_in_cache = head->mapping && !PageMappingFlags(head);Sounds good to me, but it just gets a bit complicated.
If
page_mapping()doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.
For what it's worth, I think I like Matthew's version better than the original patch. This is because, although page_mapping() looks simpler here, looking into the definition of page_mapping() I feel it's handling several cases, not all of which are relevant here (or, as Matthew points out, would actually be wrong if it were possible to reach those cases here).
It's not clear to me what is meant by "pass that over from callers"? Do you mean, have callers pass in true/false for page_in_cache directly?
That could work, but I still think I prefer Matthew's version slightly better, if only because this function already takes a lot of arguments.
Quoted 4 lines
--
Peter Xu
Peter Xu
reviewerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
Quoted 45 lines · Axel Rasmussen
Quoted 29 lines · Peter Xu
Quoted 4 lines · Matthew Wilcox
Quoted 1 line · Peter Xu
Does the patch attached look reasonable to you?
Mmm, no. If the page is in the swap cache, this will be "true".
It will not happen in practise, right?
I mean,
shmem_get_folio()should have done the swap-in, and we should have the page lock held at the meantime.For anon,
mcopy_atomic_pte()is the only user and it's passing in a newly allocated page here.Quoted 14 lines
mm/userfaultfd.c int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, · line 67
Quoted 11 lines
mm/userfaultfd.c +1
-1@@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,pte_t _dst_pte, *dst_pte;bool writable = dst_vma->vm_flags &VM_WRITE ;bool vm_shared = dst_vma->vm_flags &VM_SHARED ; -bool page_in_cache = page->mapping; +bool page_in_cache =page_mapping (page);We could do:
Quoted 7 lines
Quoted 2 lines
struct page *head = compound_head(page); bool page_in_cache = head->mapping && !PageMappingFlags(head);Sounds good to me, but it just gets a bit complicated.
If
page_mapping()doesn't sound good, how about we just pass that over from callers? We only have three, so quite doable too.For what it's worth, I think I like Matthew's version better than the original patch. This is because, although
page_mapping()looks simpler here, looking into the definition ofpage_mapping()I feel it's handling several cases, not all of which are relevant here (or, as Matthew points out, would actually be wrong if it were possible to reach those cases here).
Quoted 3 lines
It's not clear to me what is meant by "pass that over from callers"? Do you mean, have callers pass in true/false for
page_in_cachedirectly?
Yes.
Quoted 4 lines
That could work, but I still think I prefer Matthew's version slightly better, if only because this function already takes a lot of arguments.
IMHO that's not an issue, we can merge them into flags, cleaning things alongside.
The simplest so far is still just to use page_mapping() to me, but no strong opinion here.
If to go with Matthew's patch, it'll be great if we can add a comment showing what we're doing (something like "Unwrapped page_mapping() but avoid looking into swap cache" would be good enough to me).
Thanks,
Signature · 1 line
Peter Xu
Andrew Morton
maintainerRe: [PATCH 3/5] userfualtfd: Replace lru_cache functions with folio_add functions
In reply to Peter XuQuoted 12 lines · On Wed, 2 Nov 2022 15:02:35 -0400 Peter Xu <[email protected]> wrote:
mfill_atomic_install_pte()checkspage->mappingto detect whether one page is used in the page cache. However as pointed out by Matthew, the page can logically be a tail page rather than always the head in the case of uffd minor mode withUFFDIO_CONTINUE. It means we could wrongly install one pte with shmem thp tail page assuming it's an anonymous page.It's not that clear even for anonymous page, since normally anonymous pages also have
page->mappingbeing setup with the anon vma. It's safe here only because the only such caller tomfill_atomic_install_pte()is always passing in a newly allocated page (mcopy_atomic_pte()), whosepage->mappingis not yet setup. However that's not extremely obvious either.
Quoted 3 lines
For either of above, use
page_mapping()instead.And this should be stable material.
I added
PATCH 4/5 khugepage: Replace lru_cache_add() with folio_add_lru()
In reply to Vishal Moola (Oracle)Replaces some calls with their folio equivalents. This is in preparation for the removal of lru_cache_add(). This replaces 3 calls to compound_head() with 1.
--- mm/khugepaged.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
mm/khugepaged.c +7 -4
@@ -1970,6 +1970,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,if (result ==SCAN_SUCCEED ) {struct page *page, *tmp; +struct folio *folio;/* * Replacing old pages with new one has succeeded, now we
@@ -1997,11 +1998,13 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr, index++; } -SetPageUptodate (hpage); -page_ref_add (hpage,HPAGE_PMD_NR -1 ); + folio =page_folio (hpage); +folio_mark_uptodate (folio); +folio_ref_add (folio,HPAGE_PMD_NR -1 ); +if (is_shmem) -set_page_dirty (hpage); -lru_cache_add (hpage); +folio_mark_dirty (folio); +folio_add_lru (folio);/* * Remove pte page tables, so we can re-fault the page as huge.
Signature · 1 line
2.38.1
Re: [PATCH 4/5] khugepage: Replace lru_cache_add() with folio_add_lru()
Quoted 5 lines · Vishal Moola (Oracle)
Replaces some calls with their folio equivalents. This is in preparation for the removal of
lru_cache_add(). This replaces 3 calls tocompound_head()with 1.Signed-off-by Vishal Moola (Oracle) <[email protected]>
PATCH 5/5 folio-compat: Remove lru_cache_add()
In reply to Vishal Moola (Oracle)There are no longer any callers of lru_cache_add(), so remove it. This saves 107 bytes of kernel text. Also cleanup some comments such that they reference the new folio_add_lru() instead.
--- include/linux/swap.h | 1 - mm/folio-compat.c | 6 ------ mm/truncate.c | 2 +- mm/workingset.c | 2 +- 4 files changed, 2 insertions(+), 9 deletions(-)
include/linux/swap.h +0 -1
@@ -388,7 +388,6 @@ void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages);void lru_note_cost_folio (struct folio *);void folio_add_lru (struct folio *);void folio_add_lru_vma (struct folio *,struct vm_area_struct *); -void lru_cache_add (struct page *);void mark_page_accessed (struct page *);void folio_mark_accessed (struct folio *);
mm/folio-compat.c +0 -6
@@ -82,12 +82,6 @@ bool redirty_page_for_writepage(struct writeback_control *wbc, }EXPORT_SYMBOL (redirty_page_for_writepage); -void lru_cache_add (struct page *page) -{ -folio_add_lru (page_folio (page)); -} -EXPORT_SYMBOL (lru_cache_add); -void lru_cache_add_inactive_or_unevictable (struct page *page,struct vm_area_struct *vma) {
mm/truncate.c +1 -1
@@ -573,7 +573,7 @@ EXPORT_SYMBOL(invalidate_mapping_pages);* refcount. We do this because invalidate_inode_pages2() needs stronger * invalidation guarantees, and cannot afford to leave pages behind because * shrink_page_list() has a temp ref on them, or because they're transiently -* sitting in the lru_cache_add() pagevecs. +* sitting in the folio_add_lru() pagevecs. */ static int invalidate_complete_folio2 (struct address_space *mapping,struct folio *folio)
mm/workingset.c +1 -1
@@ -492,7 +492,7 @@ void workingset_refault(struct folio *folio, void *shadow)/* Folio was active prior to eviction */ if (workingset) {folio_set_workingset (folio); -/* XXX: Move to lru_cache_add() when it supports new vs putback */ +/* XXX: Move to folio_add_lru() when it supports new vs putback */ lru_note_cost_folio (folio);mod_lruvec_state (lruvec,WORKINGSET_RESTORE_BASE + file, nr); }
Signature · 1 line
2.38.1
Re: [PATCH 5/5] folio-compat: Remove lru_cache_add()
Quoted 5 lines · Vishal Moola (Oracle)
There are no longer any callers of
lru_cache_add(), so remove it. This saves 107 bytes of kernel text. Also cleanup some comments such that they reference the newfolio_add_lru()instead.Signed-off-by Vishal Moola (Oracle) <[email protected]>
Quoted 4 lines · Vishal Moola (Oracle)
This patchset replaces all calls of
lru_cache_add()with the folio
equivalent:folio_add_lru(). This is allows us to get rid of the wrapper The series passes xfstests and the userfaultfd selftests.
All of these patches have been reviewed. Andrew, is there anything you'd like to see changed, or will you take these?