better block swap batching and a different take on swap_ops v2
Subsystems (16)
Hi all,
this series makes use of the swap_iocb for block as well so that it doesn't do inefficient single-bio I/O, and then rebases the swap_ops from Baoquan on top of the now very different method structure.
When running doing kernels builds, which is a workload that doesn't really do much THP anonymous memory it still gets 2x clustering for writeout and 1.2x for reading back swap in. The overall times do not actually change, though.
Note that the buildbot is slow at the moment, so while I fixed all issues from the previous round, that run did take multiple days and the new one is still pending, so I'd rather get a v1 out now as we're most likely to have additional roundtrips anyway.
Changes since v1:
- drop the memcg accounting for fs backed swap as it can't really work as-is
- change the related refactoring to not create a bisection hazard or compile issue with the wrong set of cgroup config options enabled
- remove a now dead function
Diffstat:
Documentation/filesystems/locking.rst | 5 Documentation/filesystems/vfs.rst | 4 drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 drivers/gpu/drm/ttm/ttm_backup.c | 2 fs/nfs/file.c | 4 fs/smb/client/file.c | 4 include/linux/shmem_fs.h | 5 include/linux/swap.h | 7 include/linux/vm_event_item.h | 4 mm/madvise.c | 16 mm/page_io.c | 586 ++++++++++++++---------------- mm/shmem.c | 18 mm/swap.h | 61 +-- mm/swap_state.c | 53 +- mm/swapfile.c | 11 mm/vmscan.c | 88 ++-- mm/vmstat.c | 6 mm/zswap.c | 4 18 files changed, 450 insertions(+), 430 deletions(-)
PATCH 1/8 shmem: provide a shmem_write_folio wrapper
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O refactoring by keepin swap_iocb handling entirely contained in mm/.
--- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/ttm/ttm_backup.c | 2 +- include/linux/shmem_fs.h | 5 +---- mm/shmem.c | 7 ++++++- mm/swap.h | 4 ++++ 5 files changed, 13 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gem/i915_gem_shmem.c +1 -1
@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)if (folio_mapped (folio))folio_redirty_for_writepage (&wbc, folio);else - error =shmem_writeout (folio,NULL ,NULL ); + error =shmem_write_folio (folio); } }
drivers/gpu/drm/ttm/ttm_backup.c +1 -1
@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,if (writeback && !folio_mapped (to_folio) &&folio_clear_dirty_for_io (to_folio)) {folio_set_reclaim (to_folio); - ret =shmem_writeout (to_folio,NULL ,NULL ); + ret =shmem_write_folio (to_folio);if (!folio_test_writeback (to_folio))folio_clear_reclaim (to_folio);/*
include/linux/shmem_fs.h +1 -4
@@ -12,8 +12,6 @@#include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb ; -/* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA
@@ -122,8 +120,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)void shmem_unlock_mapping (struct address_space *mapping);struct page *shmem_read_mapping_page_gfp (struct address_space *mapping,pgoff_t index,gfp_t gfp_mask); -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, -struct list_head *folio_list); +int shmem_write_folio (struct folio *folio);void shmem_truncate_range (struct inode *inode,loff_t start,uoff_t end);int shmem_unuse (unsigned int type);
mm/shmem.c +6 -1
@@ -1738,7 +1738,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,folio_mark_dirty (folio);return AOP_WRITEPAGE_ACTIVATE ;/* Return with folio locked */ } -EXPORT_SYMBOL_GPL (shmem_writeout); + +int shmem_write_folio (struct folio *folio) +{ +return shmem_writeout (folio,NULL ,NULL ); +} +EXPORT_SYMBOL_GPL (shmem_write_folio);#if defined(CONFIG_NUMA ) && defined(CONFIG_TMPFS )static void shmem_show_mpol (struct seq_file *seq,struct mempolicy *mpol)
mm/swap.h +4 -0
@@ -473,4 +473,8 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#endif /* CONFIG_SWAP */ + +int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +struct list_head *folio_list); +#endif /* _MM_SWAP_H */
Signature · 1 line
2.53.0
Re: [PATCH 1/8] shmem: provide a shmem_write_folio wrapper
Quoted 1 line · Christoph Hellwig
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O
~~~~~ prepare, typo
Quoted 1 line
refactoring by keepin
swap_iocbhandling entirely contained inmm/.
~~~~~ keeping, typo
Quoted 9 lines
Signed-off-by Christoph Hellwig <[email protected]>--- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/ttm/ttm_backup.c | 2 +- include/linux/shmem_fs.h | 5 +---- mm/shmem.c | 7 ++++++- mm/swap.h | 4 ++++ 5 files changed, 13 insertions(+), 7 deletions(-)
Other then typos, LGTM,
Quoted 84 lines
drivers/gpu/drm/i915/gem/i915_gem_shmem.c +1
-1@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)if (folio_mapped (folio))folio_redirty_for_writepage (&wbc, folio);else - error =shmem_writeout (folio,NULL ,NULL ); + error =shmem_write_folio (folio); } }drivers/gpu/drm/ttm/ttm_backup.c +1
-1@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,if (writeback && !folio_mapped (to_folio) &&folio_clear_dirty_for_io (to_folio)) {folio_set_reclaim (to_folio); - ret =shmem_writeout (to_folio,NULL ,NULL ); + ret =shmem_write_folio (to_folio);if (!folio_test_writeback (to_folio))folio_clear_reclaim (to_folio);/* include/linux/shmem_fs.h +1
-4@@ -12,8 +12,6 @@#include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb ; -/* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -122,8 +120,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)void shmem_unlock_mapping (struct address_space *mapping);struct page *shmem_read_mapping_page_gfp (struct address_space *mapping,pgoff_t index,gfp_t gfp_mask); -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, -struct list_head *folio_list); +int shmem_write_folio (struct folio *folio);void shmem_truncate_range (struct inode *inode,loff_t start,uoff_t end);int shmem_unuse (unsigned int type);mm/shmem.c +6
-1@@ -1738,7 +1738,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,folio_mark_dirty (folio);return AOP_WRITEPAGE_ACTIVATE ;/* Return with folio locked */ } -EXPORT_SYMBOL_GPL (shmem_writeout); + +int shmem_write_folio (struct folio *folio) +{ +return shmem_writeout (folio,NULL ,NULL ); +} +EXPORT_SYMBOL_GPL (shmem_write_folio);#if defined(CONFIG_NUMA ) && defined(CONFIG_TMPFS )static void shmem_show_mpol (struct seq_file *seq,struct mempolicy *mpol)mm/swap.h +4
-0@@ -473,4 +473,8 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#endif /* CONFIG_SWAP */ + +int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +struct list_head *folio_list); +#endif /* _MM_SWAP_H */ Signature · 1 line
2.53.0
Re: [PATCH 1/8] shmem: provide a shmem_write_folio wrapper
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O refactoring by keepin
swap_iocbhandling entirely contained inmm/.Signed-off-by Christoph Hellwig <[email protected]>
better shmem writeback interface for drm, was: Re: [PATCH 1/8] shmem: provide a shmem_write_folio wrapper
In reply to Christoph HellwigSashiko complains about a lot of pre-existing issues in i915/ttm here:
https://sashiko.dev/#/patchset/20260601113449.3464734-1-hch%40lst.de
This might be a good time to fix these, and as part of it provide saner interfaces for drm. The i915 case look like a plain writeback range, which looks somewhat easy. I don't really get what the ttm code is trying to do.
Any takers to get drm out of messing with these lowlevel shmemfs internals?
Quoted 95 lines · Christoph Hellwig
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O refactoring by keepin
swap_iocbhandling entirely contained inmm/.Signed-off-by Christoph Hellwig <[email protected]>--- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/ttm/ttm_backup.c | 2 +- include/linux/shmem_fs.h | 5 +---- mm/shmem.c | 7 ++++++- mm/swap.h | 4 ++++ 5 files changed, 13 insertions(+), 7 deletions(-)drivers/gpu/drm/i915/gem/i915_gem_shmem.c +1
-1@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)if (folio_mapped (folio))folio_redirty_for_writepage (&wbc, folio);else - error =shmem_writeout (folio,NULL ,NULL ); + error =shmem_write_folio (folio); } }drivers/gpu/drm/ttm/ttm_backup.c +1
-1@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,if (writeback && !folio_mapped (to_folio) &&folio_clear_dirty_for_io (to_folio)) {folio_set_reclaim (to_folio); - ret =shmem_writeout (to_folio,NULL ,NULL ); + ret =shmem_write_folio (to_folio);if (!folio_test_writeback (to_folio))folio_clear_reclaim (to_folio);/* include/linux/shmem_fs.h +1
-4@@ -12,8 +12,6 @@#include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb ; -/* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -122,8 +120,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)void shmem_unlock_mapping (struct address_space *mapping);struct page *shmem_read_mapping_page_gfp (struct address_space *mapping,pgoff_t index,gfp_t gfp_mask); -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, -struct list_head *folio_list); +int shmem_write_folio (struct folio *folio);void shmem_truncate_range (struct inode *inode,loff_t start,uoff_t end);int shmem_unuse (unsigned int type);mm/shmem.c +6
-1@@ -1738,7 +1738,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,folio_mark_dirty (folio);return AOP_WRITEPAGE_ACTIVATE ;/* Return with folio locked */ } -EXPORT_SYMBOL_GPL (shmem_writeout); + +int shmem_write_folio (struct folio *folio) +{ +return shmem_writeout (folio,NULL ,NULL ); +} +EXPORT_SYMBOL_GPL (shmem_write_folio);#if defined(CONFIG_NUMA ) && defined(CONFIG_TMPFS )static void shmem_show_mpol (struct seq_file *seq,struct mempolicy *mpol)mm/swap.h +4
-0@@ -473,4 +473,8 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#endif /* CONFIG_SWAP */ + +int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +struct list_head *folio_list); +#endif /* _MM_SWAP_H */ Signature · 1 line
2.53.0
---end quoted text---
Re: [PATCH 1/8] shmem: provide a shmem_write_folio wrapper
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
Provide a wrapper for the shmem abuses in drm to preparare for swap I/O refactoring by keepin
swap_iocbhandling entirely contained inmm/.Signed-off-by Christoph Hellwig <[email protected]>
Chris
Quoted 91 lines
--- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/ttm/ttm_backup.c | 2 +- include/linux/shmem_fs.h | 5 +---- mm/shmem.c | 7 ++++++- mm/swap.h | 4 ++++ 5 files changed, 13 insertions(+), 7 deletions(-)drivers/gpu/drm/i915/gem/i915_gem_shmem.c +1
-1@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)if (folio_mapped (folio))folio_redirty_for_writepage (&wbc, folio);else - error =shmem_writeout (folio,NULL ,NULL ); + error =shmem_write_folio (folio); } }drivers/gpu/drm/ttm/ttm_backup.c +1
-1@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,if (writeback && !folio_mapped (to_folio) &&folio_clear_dirty_for_io (to_folio)) {folio_set_reclaim (to_folio); - ret =shmem_writeout (to_folio,NULL ,NULL ); + ret =shmem_write_folio (to_folio);if (!folio_test_writeback (to_folio))folio_clear_reclaim (to_folio);/* include/linux/shmem_fs.h +1
-4@@ -12,8 +12,6 @@#include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb ; -/* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -122,8 +120,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)void shmem_unlock_mapping (struct address_space *mapping);struct page *shmem_read_mapping_page_gfp (struct address_space *mapping,pgoff_t index,gfp_t gfp_mask); -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, -struct list_head *folio_list); +int shmem_write_folio (struct folio *folio);void shmem_truncate_range (struct inode *inode,loff_t start,uoff_t end);int shmem_unuse (unsigned int type);mm/shmem.c +6
-1@@ -1738,7 +1738,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,folio_mark_dirty (folio);return AOP_WRITEPAGE_ACTIVATE ;/* Return with folio locked */ } -EXPORT_SYMBOL_GPL (shmem_writeout); + +int shmem_write_folio (struct folio *folio) +{ +return shmem_writeout (folio,NULL ,NULL ); +} +EXPORT_SYMBOL_GPL (shmem_write_folio);#if defined(CONFIG_NUMA ) && defined(CONFIG_TMPFS )static void shmem_show_mpol (struct seq_file *seq,struct mempolicy *mpol)mm/swap.h +4
-1@@ -473,4 +473,8 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#endif /* CONFIG_SWAP */ + +int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +struct list_head *folio_list); +#endif /* _MM_SWAP_H */ --2.53.0
PATCH 2/8 mm: merge writeout into pageout
In reply to Christoph Hellwigwriteout is only called from pageout, and a straight flow at the end, so merge the two functions.
--- mm/vmscan.c | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-)
mm/vmscan.c +29 -34
@@ -612,45 +612,14 @@ typedef enum {PAGE_CLEAN , } pageout_t; -static pageout_t writeout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) -{ -int res; - -folio_set_reclaim (folio); - -/* -* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled -* or we failed to allocate contiguous swap entries, in which case -* the split out folios get added back to folio_list. -*/ -if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); -else - res =swap_writeout (folio, plug); - -if (res <0 ) -handle_write_error (mapping, folio, res); -if (res ==AOP_WRITEPAGE_ACTIVATE ) { -folio_clear_reclaim (folio); -return PAGE_ACTIVATE ; - } - -/* synchronous write? */ -if (!folio_test_writeback (folio)) -folio_clear_reclaim (folio); - -trace_mm_vmscan_write_folio (folio); -node_stat_add_folio (folio,NR_VMSCAN_WRITE ); -return PAGE_SUCCESS ; -} -/* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout (struct folio *folio,struct address_space *mapping,struct swap_iocb **plug,struct list_head *folio_list) { +int res; +/* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback.
@@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,return PAGE_ACTIVATE ;if (!folio_clear_dirty_for_io (folio))return PAGE_CLEAN ; -return writeout (folio, mapping, plug, folio_list); + +folio_set_reclaim (folio); + +/* +* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled +* or we failed to allocate contiguous swap entries, in which case +* the split out folios get added back to folio_list. +*/ +if (shmem_mapping (mapping)) + res =shmem_writeout (folio, plug, folio_list); +else + res =swap_writeout (folio, plug); + +if (res <0 ) +handle_write_error (mapping, folio, res); +if (res ==AOP_WRITEPAGE_ACTIVATE ) { +folio_clear_reclaim (folio); +return PAGE_ACTIVATE ; + } + +/* synchronous write? */ +if (!folio_test_writeback (folio)) +folio_clear_reclaim (folio); + +trace_mm_vmscan_write_folio (folio); +node_stat_add_folio (folio,NR_VMSCAN_WRITE ); +return PAGE_SUCCESS ; }/*
Signature · 1 line
2.53.0
Re: [PATCH 2/8] mm: merge writeout into pageout
Quoted 7 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>--- mm/vmscan.c | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-)
Quoted 91 lines
mm/vmscan.c +29
-34@@ -612,45 +612,14 @@ typedef enum {PAGE_CLEAN , } pageout_t; -static pageout_t writeout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) -{ -int res; - -folio_set_reclaim (folio); - -/* -* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled -* or we failed to allocate contiguous swap entries, in which case -* the split out folios get added back to folio_list. -*/ -if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); -else - res =swap_writeout (folio, plug); - -if (res <0 ) -handle_write_error (mapping, folio, res); -if (res ==AOP_WRITEPAGE_ACTIVATE ) { -folio_clear_reclaim (folio); -return PAGE_ACTIVATE ; - } - -/* synchronous write? */ -if (!folio_test_writeback (folio)) -folio_clear_reclaim (folio); - -trace_mm_vmscan_write_folio (folio); -node_stat_add_folio (folio,NR_VMSCAN_WRITE ); -return PAGE_SUCCESS ; -} -/* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout (struct folio *folio,struct address_space *mapping,struct swap_iocb **plug,struct list_head *folio_list) { +int res; +/* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback. @@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,return PAGE_ACTIVATE ;if (!folio_clear_dirty_for_io (folio))return PAGE_CLEAN ; -return writeout (folio, mapping, plug, folio_list); + +folio_set_reclaim (folio); + +/* +* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled +* or we failed to allocate contiguous swap entries, in which case +* the split out folios get added back to folio_list. +*/ +if (shmem_mapping (mapping)) + res =shmem_writeout (folio, plug, folio_list); +else + res =swap_writeout (folio, plug); + +if (res <0 ) +handle_write_error (mapping, folio, res); +if (res ==AOP_WRITEPAGE_ACTIVATE ) { +folio_clear_reclaim (folio); +return PAGE_ACTIVATE ; + } + +/* synchronous write? */ +if (!folio_test_writeback (folio)) +folio_clear_reclaim (folio); + +trace_mm_vmscan_write_folio (folio); +node_stat_add_folio (folio,NR_VMSCAN_WRITE ); +return PAGE_SUCCESS ; }/* Signature · 1 line
2.53.0
Re: [PATCH 2/8] mm: merge writeout into pageout
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>
7.2 candidate? was: Re: [PATCH 2/8] mm: merge writeout into pageout
In reply to Christoph HellwigCan we maybe pick this up for 7.2? It is a trivial cleanup that would be nice to get out of the way.
Quoted 98 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>--- mm/vmscan.c | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-)mm/vmscan.c +29
-34@@ -612,45 +612,14 @@ typedef enum {PAGE_CLEAN , } pageout_t; -static pageout_t writeout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) -{ -int res; - -folio_set_reclaim (folio); - -/* -* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled -* or we failed to allocate contiguous swap entries, in which case -* the split out folios get added back to folio_list. -*/ -if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); -else - res =swap_writeout (folio, plug); - -if (res <0 ) -handle_write_error (mapping, folio, res); -if (res ==AOP_WRITEPAGE_ACTIVATE ) { -folio_clear_reclaim (folio); -return PAGE_ACTIVATE ; - } - -/* synchronous write? */ -if (!folio_test_writeback (folio)) -folio_clear_reclaim (folio); - -trace_mm_vmscan_write_folio (folio); -node_stat_add_folio (folio,NR_VMSCAN_WRITE ); -return PAGE_SUCCESS ; -} -/* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout (struct folio *folio,struct address_space *mapping,struct swap_iocb **plug,struct list_head *folio_list) { +int res; +/* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback. @@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,return PAGE_ACTIVATE ;if (!folio_clear_dirty_for_io (folio))return PAGE_CLEAN ; -return writeout (folio, mapping, plug, folio_list); + +folio_set_reclaim (folio); + +/* +* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled +* or we failed to allocate contiguous swap entries, in which case +* the split out folios get added back to folio_list. +*/ +if (shmem_mapping (mapping)) + res =shmem_writeout (folio, plug, folio_list); +else + res =swap_writeout (folio, plug); + +if (res <0 ) +handle_write_error (mapping, folio, res); +if (res ==AOP_WRITEPAGE_ACTIVATE ) { +folio_clear_reclaim (folio); +return PAGE_ACTIVATE ; + } + +/* synchronous write? */ +if (!folio_test_writeback (folio)) +folio_clear_reclaim (folio); + +trace_mm_vmscan_write_folio (folio); +node_stat_add_folio (folio,NR_VMSCAN_WRITE ); +return PAGE_SUCCESS ; }/* Signature · 1 line
2.53.0
---end quoted text---
Re: 7.2 candidate? was: Re: [PATCH 2/8] mm: merge writeout into pageout
Quoted 89 lines · Christoph Hellwig
Can we maybe pick this up for 7.2? It is a trivial cleanup that would be nice to get out of the way.
Quoted 85 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>--- mm/vmscan.c | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-)mm/vmscan.c +22
-34@@ -612,45 +612,14 @@ typedef enum {PAGE_CLEAN , } pageout_t; -static pageout_t writeout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) -{ -int res; - -folio_set_reclaim (folio); - -/* -* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled -* or we failed to allocate contiguous swap entries, in which case -* the split out folios get added back to folio_list. -*/ -if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); -else - res =swap_writeout (folio, plug); - -if (res <0 ) -handle_write_error (mapping, folio, res); -if (res ==AOP_WRITEPAGE_ACTIVATE ) { -folio_clear_reclaim (folio); -return PAGE_ACTIVATE ; - } - -/* synchronous write? */ -if (!folio_test_writeback (folio)) -folio_clear_reclaim (folio); - -trace_mm_vmscan_write_folio (folio); -node_stat_add_folio (folio,NR_VMSCAN_WRITE ); -return PAGE_SUCCESS ; -} -/* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout (struct folio *folio,struct address_space *mapping,struct swap_iocb **plug,struct list_head *folio_list) { +int res; +/* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback. @@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,return PAGE_ACTIVATE ;if (!folio_clear_dirty_for_io (folio))return PAGE_CLEAN ; -return writeout (folio, mapping, plug, folio_list); + +folio_set_reclaim (folio); + +/* +* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled +* or we failed to allocate contiguous swap entries, in which case +* the split out folios get added back to folio_list. +*/ +if (shmem_mapping (mapping)) + res =shmem_writeout (folio, plug, folio_list); +else + res =swap_writeout (folio, plug); + +if (res <0 ) +handle_write_error (mapping, folio, res); +if (res ==AOP_WRITEPAGE_ACTIVATE ) { +folio_clear_reclaim (folio); +return PAGE_ACTIVATE ; + } +
Quoted 8 lines
mm/vmscan.c static pageout_t pageout(struct folio *folio, struct address_space *mapping, · line 673
Quoted 8 lines
mm/vmscan.c +7
-0+/* synchronous write? */ +if (!folio_test_writeback (folio)) +folio_clear_reclaim (folio); + +trace_mm_vmscan_write_folio (folio); +node_stat_add_folio (folio,NR_VMSCAN_WRITE ); +return PAGE_SUCCESS ; }
LGTM
It's rather trivial, so throwing it into mm-unstable for a bit to merge it in the second MM PULL would make sense to me.
Signature · 3 lines
Cheers, David
Re: [PATCH 2/8] mm: merge writeout into pageout
In reply to Christoph HellwigQuoted 4 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>
LGTM for 7.2 inclusion.
Re: [PATCH 2/8] mm: merge writeout into pageout
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
writeout is only called from pageout, and a straight flow at the end, so merge the two functions.
Signed-off-by Christoph Hellwig <[email protected]>
Chris
Quoted 94 lines
--- mm/vmscan.c | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-)mm/vmscan.c +29
-35@@ -612,45 +612,14 @@ typedef enum {PAGE_CLEAN , } pageout_t; -static pageout_t writeout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) -{ -int res; - -folio_set_reclaim (folio); - -/* -* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled -* or we failed to allocate contiguous swap entries, in which case -* the split out folios get added back to folio_list. -*/ -if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); -else - res =swap_writeout (folio, plug); - -if (res <0 ) -handle_write_error (mapping, folio, res); -if (res ==AOP_WRITEPAGE_ACTIVATE ) { -folio_clear_reclaim (folio); -return PAGE_ACTIVATE ; - } - -/* synchronous write? */ -if (!folio_test_writeback (folio)) -folio_clear_reclaim (folio); - -trace_mm_vmscan_write_folio (folio); -node_stat_add_folio (folio,NR_VMSCAN_WRITE ); -return PAGE_SUCCESS ; -} -/* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout (struct folio *folio,struct address_space *mapping,struct swap_iocb **plug,struct list_head *folio_list) { +int res; +/* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback. @@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,return PAGE_ACTIVATE ;if (!folio_clear_dirty_for_io (folio))return PAGE_CLEAN ; -return writeout (folio, mapping, plug, folio_list); + +folio_set_reclaim (folio); + +/* +* The large shmem folio can be split if CONFIG_THP_SWAP is not enabled +* or we failed to allocate contiguous swap entries, in which case +* the split out folios get added back to folio_list. +*/ +if (shmem_mapping (mapping)) + res =shmem_writeout (folio, plug, folio_list); +else + res =swap_writeout (folio, plug); + +if (res <0 ) +handle_write_error (mapping, folio, res); +if (res ==AOP_WRITEPAGE_ACTIVATE ) { +folio_clear_reclaim (folio); +return PAGE_ACTIVATE ; + } + +/* synchronous write? */ +if (!folio_test_writeback (folio)) +folio_clear_reclaim (folio); + +trace_mm_vmscan_write_folio (folio); +node_stat_add_folio (folio,NR_VMSCAN_WRITE ); +return PAGE_SUCCESS ; }/* -- 2.53.0
PATCH 3/8 mm/swap: introduce struct swap_io_ctx
In reply to Christoph HellwigGeneralize the context currently provided by double pointers to struct swap_iocb to an on-stack context. This cleans up the code and prepares for adding more fields and supporting batching multiple folios into a single bio for block-based swap as well.
This new swap_io_ctx is required for all functions using it, the old way of allowing a NULL iocb for some callers is removed to keep the interface consistent.
--- mm/madvise.c | 16 ++++++------- mm/page_io.c | 60 ++++++++++++++++++++++++++----------------------- mm/shmem.c | 13 +++++++---- mm/swap.h | 36 +++++++++++++---------------- mm/swap_state.c | 53 +++++++++++++++++++++++++------------------ mm/vmscan.c | 15 ++++++------- mm/zswap.c | 4 +++- 7 files changed, 106 insertions(+), 91 deletions(-)
mm/madvise.c +8 -8
@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};pte_t *ptep =NULL ;spinlock_t *ptl;unsigned long addr;
@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,pte_unmap_unlock (ptep, ptl); ptep =NULL ; - folio =read_swap_cache_async (entry,GFP_HIGHUSER_MOVABLE , - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry,GFP_HIGHUSER_MOVABLE , + vma, addr);if (folio)folio_put (folio); }if (ptep)pte_unmap_unlock (ptep, ptl); -swap_read_unplug (splug); +swap_read_submit (&ctx);cond_resched ();return 0 ;
@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,XA_STATE (xas, &mapping->i_pages,linear_page_index (vma, start));pgoff_t end_index =linear_page_index (vma, end) -1 ;struct folio *folio; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};rcu_read_lock ();xas_for_each (&xas, folio, end_index) {
@@ -257,15 +257,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma,xas_pause (&xas);rcu_read_unlock (); - folio =read_swap_cache_async (entry,mapping_gfp_mask (mapping), - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry, +mapping_gfp_mask (mapping), vma, addr);if (folio)folio_put (folio); rcu_read_lock(); }rcu_read_unlock (); -swap_read_unplug (splug); +swap_read_submit (&ctx); }#endif /* CONFIG_SWAP */
mm/page_io.c +32 -28
162 lines not on this page · raw message
mm/shmem.c +9 -4
@@ -1584,13 +1584,13 @@ int shmem_unuse(unsigned int type)/** * shmem_writeout - Write the folio to swap +* @ctx: swap I/O context * @folio: The folio to write -* @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list) {struct address_space *mapping = folio->mapping;
@@ -1702,7 +1702,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,shmem_delete_from_page_cache (folio,swp_to_radix_entry (folio->swap));BUG_ON (folio_mapped (folio)); - error =swap_writeout (folio, plug); + error =swap_writeout (ctx, folio);if (error !=AOP_WRITEPAGE_ACTIVATE ) {/* folio has been unlocked */ return error;
@@ -1741,7 +1741,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,int shmem_write_folio (struct folio *folio) { -return shmem_writeout (folio,NULL ,NULL ); +struct swap_io_ctx ctx = {}; +int err; + + err =shmem_writeout (&ctx, folio,NULL ); +swap_write_submit (&ctx); +return err; }EXPORT_SYMBOL_GPL (shmem_write_folio);
mm/swap.h +16 -20
@@ -4,6 +4,7 @@#include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ +struct mempolicy ;struct swap_iocb ;struct swap_memcg_table ;
@@ -78,6 +79,10 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +struct swap_io_ctx { +struct swap_iocb *sio; +}; +#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */
@@ -240,17 +245,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,/* linux/mm/page_io.c */ int sio_pool_init (void ); -struct swap_iocb ; -void swap_read_folio (struct folio *folio,struct swap_iocb **plug); -void __swap_read_unplug (struct swap_iocb *plug); -static inline void swap_read_unplug (struct swap_iocb *plug) -{ -if (unlikely (plug)) -__swap_read_unplug (plug); -} -void swap_write_unplug (struct swap_iocb *sio); -int swap_writeout (struct folio *folio,struct swap_iocb **swap_plug); -void __swap_writepage (struct folio *folio,struct swap_iocb **swap_plug); +void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio); +void swap_read_submit (struct swap_io_ctx *ctx); +void swap_write_submit (struct swap_io_ctx *ctx); +int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio); +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio);/* linux/mm/swap_state.c */ extern struct address_space swap_space__read_mostly ;
@@ -317,9 +316,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,void show_swap_cache_info (void );void swapcache_clear (struct swap_info_struct *si,swp_entry_t entry,int nr); -struct folio *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask, -struct vm_area_struct *vma,unsigned long addr, -struct swap_iocb **plug); +struct folio *read_swap_cache_async (struct swap_io_ctx *ctx,swp_entry_t entry, +gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr);struct folio *swap_cluster_readahead (swp_entry_t entry,gfp_t flag,struct mempolicy *mpol,pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
@@ -335,7 +333,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#else /* CONFIG_SWAP */ -struct swap_iocb ;static inline struct swap_cluster_info *swap_cluster_lock (struct swap_info_struct *si,pgoff_t offset,bool irq) {
@@ -381,11 +378,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio (struct folio *folio,struct swap_iocb **plug) +static inline void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) { } -static inline void swap_write_unplug (struct swap_iocb *sio) +static inline void swap_write_submit (struct swap_io_ctx *ctx) { }
@@ -427,8 +424,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout (struct folio *folio, -struct swap_iocb **swap_plug) +static inline int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio) {return 0 ; }
@@ -474,7 +470,7 @@ static inline unsigned int folio_swap_flags(struct folio *folio)#endif /* CONFIG_SWAP */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);#endif /* _MM_SWAP_H */
mm/swap_state.c +31 -22
137 lines not on this page · raw message
mm/vmscan.c +7 -8
@@ -615,8 +615,8 @@ typedef enum {/* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) +static pageout_t pageout (struct swap_io_ctx *ctx,struct address_space *mapping, +struct folio *folio,struct list_head *folio_list) {int res;
@@ -652,9 +652,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,* the split out folios get added back to folio_list. */ if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); + res =shmem_writeout (ctx, folio, folio_list);else - res =swap_writeout (folio, plug); + res =swap_writeout (ctx, folio);if (res <0 )handle_write_error (mapping, folio, res);
@@ -1063,7 +1063,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,unsigned int nr_reclaimed =0 , nr_demoted =0 ;unsigned int pgactivate =0 ;bool do_demote_pass; -struct swap_iocb *plug =NULL ; +struct swap_io_ctx ctx = {};folio_batch_init (&free_folios);memset (stat,0 ,sizeof (*stat));
@@ -1394,7 +1394,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,* starts and then write it out here. */ try_to_unmap_flush_dirty (); -switch (pageout (folio, mapping, &plug, folio_list)) { +switch (pageout (&ctx, mapping, folio, folio_list)) {case PAGE_KEEP : goto keep_locked;case PAGE_ACTIVATE :
@@ -1584,8 +1584,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,list_splice (&ret_folios, folio_list);count_vm_events (PGACTIVATE , pgactivate); -if (plug) -swap_write_unplug (plug); +swap_write_submit (&ctx);return nr_reclaimed; }
mm/zswap.c +3 -1
@@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,struct folio *folio;struct mempolicy *mpol;struct swap_info_struct *si; +struct swap_io_ctx ctx = {};int ret =0 ;/* try to allocate swap cache folio */
@@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,folio_set_reclaim (folio);/* start writeback */ -__swap_writepage (folio,NULL ); +__swap_writepage (&ctx, folio); +swap_write_submit (&ctx); out:if (ret) {
Signature · 1 line
2.53.0
Baoquan He
reviewerRe: [PATCH 3/8] mm/swap: introduce struct swap_io_ctx
Quoted 8 lines · Christoph Hellwig
Generalize the context currently provided by double pointers to struct
swap_iocbto an on-stack context. This cleans up the code and prepares for adding more fields and supporting batching multiple folios into a single bio for block-based swap as well.This new
swap_io_ctxis required for all functions using it, the old way of allowing a NULL iocb for some callers is removed to keep the interface consistent.
In this patch, except of introducng struct swap_io_ctx, it also does:
- swap_cache_read_folio_sync() helper introduced for the two readahead "skip" paths where the target page was already read
- swap_write_unplug → swap_write_submit,
__swap_read_unplug → swap_read_submit;
the NULL-checking inline swap_read_unplug() wrapper is removed
It would be better if these are mentioned in log too, or should be split into a separate patch. Otherwise it doesn't match the subject.
Quoted 26 lines
Signed-off-by Christoph Hellwig <[email protected]>--- mm/madvise.c | 16 ++++++------- mm/page_io.c | 60 ++++++++++++++++++++++++++----------------------- mm/shmem.c | 13 +++++++---- mm/swap.h | 36 +++++++++++++---------------- mm/swap_state.c | 53 +++++++++++++++++++++++++------------------ mm/vmscan.c | 15 ++++++------- mm/zswap.c | 4 +++- 7 files changed, 106 insertions(+), 91 deletions(-)mm/madvise.c +1
-1@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};pte_t *ptep =NULL ;spinlock_t *ptl;unsigned long addr;
mm/madvise.c static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, · line 216
Quoted 7 lines
mm/madvise.c +2
-2@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,pte_unmap_unlock (ptep, ptl); ptep =NULL ; - folio =read_swap_cache_async (entry,GFP_HIGHUSER_MOVABLE , - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry,GFP_HIGHUSER_MOVABLE , + vma, addr);
And I am wondering why &ctx has to be put as the 1st para, is there a rule of C language or linux kernel? It obviously adds a lot of unnecessary code churn. I will read swap cache, of course swap entry is the main subject I will operate on, why bother moving &ctx which is associated date at first.
Quoted 592 lines
mm/madvise.c +5
-5if (folio)folio_put (folio); }if (ptep)pte_unmap_unlock (ptep, ptl); -swap_read_unplug (splug); +swap_read_submit (&ctx);cond_resched ();return 0 ;@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,XA_STATE (xas, &mapping->i_pages,linear_page_index (vma, start));pgoff_t end_index =linear_page_index (vma, end) -1 ;struct folio *folio; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};rcu_read_lock ();xas_for_each (&xas, folio, end_index) {@@ -257,15 +257,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma,xas_pause (&xas);rcu_read_unlock (); - folio =read_swap_cache_async (entry,mapping_gfp_mask (mapping), - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry, +mapping_gfp_mask (mapping), vma, addr);if (folio)folio_put (folio); rcu_read_lock(); }rcu_read_unlock (); -swap_read_unplug (splug); +swap_read_submit (&ctx); }#endif /* CONFIG_SWAP */ mm/page_io.c +32
-28@@ -248,7 +248,7 @@ static void swap_zeromap_folio_clear(struct folio *folio)* We may have stale swap cache pages in memory: notice * them here and get rid of the unnecessary final write. */ -int swap_writeout (struct folio *folio,struct swap_iocb **swap_plug) +int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio) {int ret =0 ;@@ -295,7 +295,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) }rcu_read_unlock (); -__swap_writepage (folio, swap_plug); +__swap_writepage (ctx, folio);return 0 ; out_unlock:folio_unlock (folio);@@ -385,9 +385,9 @@ static void sio_write_complete(struct kiocb *iocb, long ret)mempool_free (sio, sio_pool); } -static void swap_writepage_fs (struct folio *folio,struct swap_iocb **swap_plug) +static void swap_writepage_fs (struct swap_io_ctx *ctx,struct folio *folio) { -struct swap_iocb *sio = swap_plug ? *swap_plug :NULL ; +struct swap_iocb *sio = ctx->sio;struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);struct file *swap_file = sis->swap_file;loff_t pos =swap_dev_pos (folio->swap);@@ -398,7 +398,7 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)if (sio) {if (sio->iocb.ki_filp != swap_file || sio->iocb.ki_pos + sio->len != pos) { -swap_write_unplug (sio); +swap_write_submit (ctx); sio =NULL ; } }@@ -413,12 +413,11 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs) || !swap_plug) { -swap_write_unplug (sio); +if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +swap_write_submit (ctx); sio =NULL ; } -if (swap_plug) - *swap_plug = sio; + ctx->sio = sio; } static void swap_writepage_bdev_sync(struct folio *folio,@@ -458,7 +457,7 @@ static void swap_writepage_bdev_async(struct folio *folio,submit_bio (bio); } -void __swap_writepage (struct folio *folio,struct swap_iocb **swap_plug) +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);@@ -469,7 +468,7 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)* is safe. */ if (data_race (sis->flags &SWP_FS_OPS )) -swap_writepage_fs (folio, swap_plug); +swap_writepage_fs (ctx, folio);/* * ->flags can be updated non-atomically, * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race @@ -481,16 +480,20 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)swap_writepage_bdev_async (folio, sis); } -void swap_write_unplug (struct swap_iocb *sio) +void swap_write_submit (struct swap_io_ctx *ctx) { +struct swap_iocb *sio = ctx->sio;struct iov_iter from; -struct address_space *mapping = sio->iocb.ki_filp->f_mapping;int ret; +if (!sio) +return ; +iov_iter_bvec (&from,ITER_SOURCE , sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw (&sio->iocb, &from); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from);if (ret != -EIOCBQUEUED )sio_write_complete (&sio->iocb, ret); + ctx->sio =NULL ; }static void sio_read_complete (struct kiocb *iocb,long ret)@@ -582,18 +585,16 @@ static bool swap_read_folio_zeromap(struct folio *folio)return true; } -static void swap_read_folio_fs (struct folio *folio,struct swap_iocb **plug) +static void swap_read_folio_fs (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct swap_iocb *sio =NULL ; +struct swap_iocb *sio = ctx->sio;loff_t pos =swap_dev_pos (folio->swap); -if (plug) - sio = *plug;if (sio) {if (sio->iocb.ki_filp != sis->swap_file || sio->iocb.ki_pos + sio->len != pos) { -swap_read_unplug (sio); +swap_read_submit (ctx); sio =NULL ; } }@@ -608,12 +609,11 @@ static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs) || !plug) { -swap_read_unplug (sio); +if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +swap_read_submit (ctx); sio =NULL ; } -if (plug) - *plug = sio; + ctx->sio = sio; } static void swap_read_folio_bdev_sync(struct folio *folio,@@ -653,7 +653,7 @@ static void swap_read_folio_bdev_async(struct folio *folio,submit_bio (bio); } -void swap_read_folio (struct folio *folio,struct swap_iocb **plug) +void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);bool synchronous = sis->flags &SWP_SYNCHRONOUS_IO ;@@ -688,7 +688,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)zswap_folio_swapin (folio);if (data_race (sis->flags &SWP_FS_OPS )) { -swap_read_folio_fs (folio, plug); +swap_read_folio_fs (ctx, folio); }else if (synchronous) {swap_read_folio_bdev_sync (folio, sis); }else {@@ -703,14 +703,18 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)delayacct_swapin_end (); } -void __swap_read_unplug (struct swap_iocb *sio) +void swap_read_submit (struct swap_io_ctx *ctx) { +struct swap_iocb *sio = ctx->sio;struct iov_iter from; -struct address_space *mapping = sio->iocb.ki_filp->f_mapping;int ret; +if (!sio) +return ; +iov_iter_bvec (&from,ITER_DEST , sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw (&sio->iocb, &from); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from);if (ret != -EIOCBQUEUED )sio_read_complete (&sio->iocb, ret); + ctx->sio =NULL ; }mm/shmem.c +9
-4@@ -1584,13 +1584,13 @@ int shmem_unuse(unsigned int type)/** * shmem_writeout - Write the folio to swap +* @ctx: swap I/O context * @folio: The folio to write -* @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list) {struct address_space *mapping = folio->mapping;@@ -1702,7 +1702,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,shmem_delete_from_page_cache (folio,swp_to_radix_entry (folio->swap));BUG_ON (folio_mapped (folio)); - error =swap_writeout (folio, plug); + error =swap_writeout (ctx, folio);if (error !=AOP_WRITEPAGE_ACTIVATE ) {/* folio has been unlocked */ return error;@@ -1741,7 +1741,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,int shmem_write_folio (struct folio *folio) { -return shmem_writeout (folio,NULL ,NULL ); +struct swap_io_ctx ctx = {}; +int err; + + err =shmem_writeout (&ctx, folio,NULL ); +swap_write_submit (&ctx); +return err; }EXPORT_SYMBOL_GPL (shmem_write_folio);mm/swap.h +16
-20@@ -4,6 +4,7 @@#include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ +struct mempolicy ;struct swap_iocb ;struct swap_memcg_table ;@@ -78,6 +79,10 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +struct swap_io_ctx { +struct swap_iocb *sio; +}; +#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ @@ -240,17 +245,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,/* linux/mm/page_io.c */ int sio_pool_init (void ); -struct swap_iocb ; -void swap_read_folio (struct folio *folio,struct swap_iocb **plug); -void __swap_read_unplug (struct swap_iocb *plug); -static inline void swap_read_unplug (struct swap_iocb *plug) -{ -if (unlikely (plug)) -__swap_read_unplug (plug); -} -void swap_write_unplug (struct swap_iocb *sio); -int swap_writeout (struct folio *folio,struct swap_iocb **swap_plug); -void __swap_writepage (struct folio *folio,struct swap_iocb **swap_plug); +void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio); +void swap_read_submit (struct swap_io_ctx *ctx); +void swap_write_submit (struct swap_io_ctx *ctx); +int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio); +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio);/* linux/mm/swap_state.c */ extern struct address_space swap_space__read_mostly ;@@ -317,9 +316,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,void show_swap_cache_info (void );void swapcache_clear (struct swap_info_struct *si,swp_entry_t entry,int nr); -struct folio *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask, -struct vm_area_struct *vma,unsigned long addr, -struct swap_iocb **plug); +struct folio *read_swap_cache_async (struct swap_io_ctx *ctx,swp_entry_t entry, +gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr);struct folio *swap_cluster_readahead (swp_entry_t entry,gfp_t flag,struct mempolicy *mpol,pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,@@ -335,7 +333,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#else /* CONFIG_SWAP */ -struct swap_iocb ;static inline struct swap_cluster_info *swap_cluster_lock (struct swap_info_struct *si,pgoff_t offset,bool irq) {@@ -381,11 +378,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio (struct folio *folio,struct swap_iocb **plug) +static inline void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) { } -static inline void swap_write_unplug (struct swap_iocb *sio) +static inline void swap_write_submit (struct swap_io_ctx *ctx) { }@@ -427,8 +424,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout (struct folio *folio, -struct swap_iocb **swap_plug) +static inline int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio) {return 0 ; }@@ -474,7 +470,7 @@ static inline unsigned int folio_swap_flags(struct folio *folio)#endif /* CONFIG_SWAP */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);#endif /* _MM_SWAP_H */ mm/swap_state.c +31
-22@@ -623,9 +623,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, } } -static struct folio *swap_cache_read_folio (swp_entry_t entry,gfp_t gfp, -struct mempolicy *mpol,pgoff_t ilx, -struct swap_iocb **plug,bool readahead) +static struct folio *swap_cache_read_folio (struct swap_io_ctx *ctx, +swp_entry_t entry,gfp_t gfp,struct mempolicy *mpol, +pgoff_t ilx,bool readahead) {struct folio *folio;@@ -639,7 +639,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,if (IS_ERR_OR_NULL (folio))return NULL ; -swap_read_folio (folio, plug); +swap_read_folio (ctx, folio);if (readahead) {folio_set_readahead (folio);count_vm_event (SWAP_RA );@@ -667,6 +667,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,struct folio *swapin_sync (swp_entry_t entry,gfp_t gfp,unsigned long orders,struct vm_fault *vmf,struct mempolicy *mpol,pgoff_t ilx) { +struct swap_io_ctx ctx = {};struct folio *folio; do {@@ -679,7 +680,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,if (IS_ERR (folio))return folio; -swap_read_folio (folio,NULL ); +swap_read_folio (&ctx, folio); +swap_read_submit (&ctx);return folio; }@@ -689,9 +691,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,* A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct folio *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask, -struct vm_area_struct *vma,unsigned long addr, -struct swap_iocb **plug) +struct folio *read_swap_cache_async (struct swap_io_ctx *ctx,swp_entry_t entry, +gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr) {struct swap_info_struct *si;struct mempolicy *mpol;@@ -703,13 +704,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,return NULL ; mpol =get_vma_policy (vma, addr,0 , &ilx); - folio =swap_cache_read_folio (entry, gfp_mask, mpol, ilx, plug, false); + folio =swap_cache_read_folio (ctx, entry, gfp_mask, mpol, ilx, false);mpol_cond_put (mpol);put_swap_device (si);return folio; } +static struct folio *swap_cache_read_folio_sync (swp_entry_t entry,gfp_t gfp, +struct mempolicy *mpol,pgoff_t ilx) +{ +struct swap_io_ctx ctx = {}; +struct folio *folio; + + folio =swap_cache_read_folio (&ctx, entry, gfp, mpol, ilx, false); +swap_read_submit (&ctx); +return folio; +} + static unsigned int __swapin_nr_pages(unsigned long prev_offset, unsigned long offset, int hits,@@ -798,8 +810,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,unsigned long start_offset, end_offset;unsigned long mask;struct swap_info_struct *si =__swap_entry_to_info (entry); +struct swap_io_ctx ctx = {};struct blk_plug plug; -struct swap_iocb *splug =NULL ;swp_entry_t ra_entry; mask =swapin_nr_pages (offset) -1 ;@@ -818,18 +830,17 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,for (offset = start_offset; offset <= end_offset ; offset++) {/* Ok, do the async read-ahead now */ ra_entry =swp_entry (swp_type (entry), offset); - folio =swap_cache_read_folio (ra_entry, gfp_mask, mpol, ilx, - &splug, offset != entry_offset); + folio =swap_cache_read_folio (&ctx, ra_entry, gfp_mask, mpol, + ilx, offset != entry_offset);if (!folio)continue ;folio_put (folio); }blk_finish_plug (&plug); -swap_read_unplug (splug); +swap_read_submit (&ctx);lru_add_drain ();/* Push any new pages onto the LRU now */ skip: -/* The page was likely read above, so no need for plugging here */ -return swap_cache_read_folio (entry, gfp_mask, mpol, ilx,NULL , false); +return swap_cache_read_folio_sync (entry, gfp_mask, mpol, ilx); } static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,@@ -889,8 +900,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,static struct folio *swap_vma_readahead (swp_entry_t targ_entry,gfp_t gfp_mask,struct mempolicy *mpol,pgoff_t targ_ilx,struct vm_fault *vmf) { +struct swap_io_ctx ctx = {};struct blk_plug plug; -struct swap_iocb *splug =NULL ;struct folio *folio;pte_t *pte =NULL , pentry;int win;@@ -929,8 +940,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,if (!si)continue ; } - folio =swap_cache_read_folio (entry, gfp_mask, mpol, ilx, - &splug, addr != vmf->address); + folio =swap_cache_read_folio (&ctx, entry, gfp_mask, mpol, ilx, + addr != vmf->address);if (si)put_swap_device (si);if (!folio)@@ -940,13 +951,11 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,if (pte)pte_unmap (pte);blk_finish_plug (&plug); -swap_read_unplug (splug); +swap_read_submit (&ctx);lru_add_drain (); skip:/* The folio was likely read above, so no need for plugging here */ - folio =swap_cache_read_folio (targ_entry, gfp_mask, mpol, targ_ilx, -NULL , false); -return folio; +return swap_cache_read_folio_sync (targ_entry, gfp_mask, mpol, targ_ilx); }/** mm/vmscan.c +7
-8@@ -615,8 +615,8 @@ typedef enum {/* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) +static pageout_t pageout (struct swap_io_ctx *ctx,struct address_space *mapping, +struct folio *folio,struct list_head *folio_list) {int res;@@ -652,9 +652,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,* the split out folios get added back to folio_list. */ if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); + res =shmem_writeout (ctx, folio, folio_list);else - res =swap_writeout (folio, plug); + res =swap_writeout (ctx, folio);if (res <0 )handle_write_error (mapping, folio, res);@@ -1063,7 +1063,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,unsigned int nr_reclaimed =0 , nr_demoted =0 ;unsigned int pgactivate =0 ;bool do_demote_pass; -struct swap_iocb *plug =NULL ; +struct swap_io_ctx ctx = {};folio_batch_init (&free_folios);memset (stat,0 ,sizeof (*stat));@@ -1394,7 +1394,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,* starts and then write it out here. */ try_to_unmap_flush_dirty (); -switch (pageout (folio, mapping, &plug, folio_list)) { +switch (pageout (&ctx, mapping, folio, folio_list)) {case PAGE_KEEP : goto keep_locked;case PAGE_ACTIVATE :@@ -1584,8 +1584,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,list_splice (&ret_folios, folio_list);count_vm_events (PGACTIVATE , pgactivate); -if (plug) -swap_write_unplug (plug); +swap_write_submit (&ctx);return nr_reclaimed; }mm/zswap.c +3
-1@@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,struct folio *folio;struct mempolicy *mpol;struct swap_info_struct *si; +struct swap_io_ctx ctx = {};int ret =0 ;/* try to allocate swap cache folio */ @@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,folio_set_reclaim (folio);/* start writeback */ -__swap_writepage (folio,NULL ); +__swap_writepage (&ctx, folio); +swap_write_submit (&ctx); out:if (ret) {Signature · 1 line
2.53.0
Re: [PATCH 3/8] mm/swap: introduce struct swap_io_ctx
In reply to Christoph HellwigQuoted 622 lines · Christoph Hellwig
Generalize the context currently provided by double pointers to struct
swap_iocbto an on-stack context. This cleans up the code and prepares for adding more fields and supporting batching multiple folios into a single bio for block-based swap as well.This new
swap_io_ctxis required for all functions using it, the old way of allowing a NULL iocb for some callers is removed to keep the interface consistent.Signed-off-by Christoph Hellwig <[email protected]>--- mm/madvise.c | 16 ++++++------- mm/page_io.c | 60 ++++++++++++++++++++++++++----------------------- mm/shmem.c | 13 +++++++---- mm/swap.h | 36 +++++++++++++---------------- mm/swap_state.c | 53 +++++++++++++++++++++++++------------------ mm/vmscan.c | 15 ++++++------- mm/zswap.c | 4 +++- 7 files changed, 106 insertions(+), 91 deletions(-)mm/madvise.c +8
-8@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *walk) { struct vm_area_struct *vma = walk->private; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};pte_t *ptep =NULL ;spinlock_t *ptl;unsigned long addr;@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,pte_unmap_unlock (ptep, ptl); ptep =NULL ; - folio =read_swap_cache_async (entry,GFP_HIGHUSER_MOVABLE , - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry,GFP_HIGHUSER_MOVABLE , + vma, addr);if (folio)folio_put (folio); }if (ptep)pte_unmap_unlock (ptep, ptl); -swap_read_unplug (splug); +swap_read_submit (&ctx);cond_resched ();return 0 ;@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,XA_STATE (xas, &mapping->i_pages,linear_page_index (vma, start));pgoff_t end_index =linear_page_index (vma, end) -1 ;struct folio *folio; -struct swap_iocb *splug =NULL ; +struct swap_io_ctx ctx = {};rcu_read_lock ();xas_for_each (&xas, folio, end_index) {@@ -257,15 +257,15 @@ static void shmem_swapin_range(struct vm_area_struct *vma,xas_pause (&xas);rcu_read_unlock (); - folio =read_swap_cache_async (entry,mapping_gfp_mask (mapping), - vma, addr, &splug); + folio =read_swap_cache_async (&ctx, entry, +mapping_gfp_mask (mapping), vma, addr);if (folio)folio_put (folio); rcu_read_lock(); }rcu_read_unlock (); -swap_read_unplug (splug); +swap_read_submit (&ctx); }#endif /* CONFIG_SWAP */ mm/page_io.c +32
-28@@ -248,7 +248,7 @@ static void swap_zeromap_folio_clear(struct folio *folio)* We may have stale swap cache pages in memory: notice * them here and get rid of the unnecessary final write. */ -int swap_writeout (struct folio *folio,struct swap_iocb **swap_plug) +int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio) {int ret =0 ;@@ -295,7 +295,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) }rcu_read_unlock (); -__swap_writepage (folio, swap_plug); +__swap_writepage (ctx, folio);return 0 ; out_unlock:folio_unlock (folio);@@ -385,9 +385,9 @@ static void sio_write_complete(struct kiocb *iocb, long ret)mempool_free (sio, sio_pool); } -static void swap_writepage_fs (struct folio *folio,struct swap_iocb **swap_plug) +static void swap_writepage_fs (struct swap_io_ctx *ctx,struct folio *folio) { -struct swap_iocb *sio = swap_plug ? *swap_plug :NULL ; +struct swap_iocb *sio = ctx->sio;struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);struct file *swap_file = sis->swap_file;loff_t pos =swap_dev_pos (folio->swap);@@ -398,7 +398,7 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)if (sio) {if (sio->iocb.ki_filp != swap_file || sio->iocb.ki_pos + sio->len != pos) { -swap_write_unplug (sio); +swap_write_submit (ctx); sio =NULL ; } }@@ -413,12 +413,11 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs) || !swap_plug) { -swap_write_unplug (sio); +if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +swap_write_submit (ctx); sio =NULL ; } -if (swap_plug) - *swap_plug = sio; + ctx->sio = sio; } static void swap_writepage_bdev_sync(struct folio *folio,@@ -458,7 +457,7 @@ static void swap_writepage_bdev_async(struct folio *folio,submit_bio (bio); } -void __swap_writepage (struct folio *folio,struct swap_iocb **swap_plug) +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);@@ -469,7 +468,7 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)* is safe. */ if (data_race (sis->flags &SWP_FS_OPS )) -swap_writepage_fs (folio, swap_plug); +swap_writepage_fs (ctx, folio);/* * ->flags can be updated non-atomically, * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race @@ -481,16 +480,20 @@ void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)swap_writepage_bdev_async (folio, sis); } -void swap_write_unplug (struct swap_iocb *sio) +void swap_write_submit (struct swap_io_ctx *ctx) { +struct swap_iocb *sio = ctx->sio;struct iov_iter from; -struct address_space *mapping = sio->iocb.ki_filp->f_mapping;int ret; +if (!sio) +return ; +iov_iter_bvec (&from,ITER_SOURCE , sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw (&sio->iocb, &from); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from);if (ret != -EIOCBQUEUED )sio_write_complete (&sio->iocb, ret); + ctx->sio =NULL ; }static void sio_read_complete (struct kiocb *iocb,long ret)@@ -582,18 +585,16 @@ static bool swap_read_folio_zeromap(struct folio *folio)return true; } -static void swap_read_folio_fs (struct folio *folio,struct swap_iocb **plug) +static void swap_read_folio_fs (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct swap_iocb *sio =NULL ; +struct swap_iocb *sio = ctx->sio;loff_t pos =swap_dev_pos (folio->swap); -if (plug) - sio = *plug;if (sio) {if (sio->iocb.ki_filp != sis->swap_file || sio->iocb.ki_pos + sio->len != pos) { -swap_read_unplug (sio); +swap_read_submit (ctx); sio =NULL ; } }@@ -608,12 +609,11 @@ static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs) || !plug) { -swap_read_unplug (sio); +if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +swap_read_submit (ctx); sio =NULL ; } -if (plug) - *plug = sio; + ctx->sio = sio; } static void swap_read_folio_bdev_sync(struct folio *folio,@@ -653,7 +653,7 @@ static void swap_read_folio_bdev_async(struct folio *folio,submit_bio (bio); } -void swap_read_folio (struct folio *folio,struct swap_iocb **plug) +void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);bool synchronous = sis->flags &SWP_SYNCHRONOUS_IO ;@@ -688,7 +688,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)zswap_folio_swapin (folio);if (data_race (sis->flags &SWP_FS_OPS )) { -swap_read_folio_fs (folio, plug); +swap_read_folio_fs (ctx, folio); }else if (synchronous) {swap_read_folio_bdev_sync (folio, sis); }else {@@ -703,14 +703,18 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)delayacct_swapin_end (); } -void __swap_read_unplug (struct swap_iocb *sio) +void swap_read_submit (struct swap_io_ctx *ctx) { +struct swap_iocb *sio = ctx->sio;struct iov_iter from; -struct address_space *mapping = sio->iocb.ki_filp->f_mapping;int ret; +if (!sio) +return ; +iov_iter_bvec (&from,ITER_DEST , sio->bvecs, sio->nr_bvecs, sio->len); - ret = mapping->a_ops->swap_rw (&sio->iocb, &from); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from);if (ret != -EIOCBQUEUED )sio_read_complete (&sio->iocb, ret); + ctx->sio =NULL ; }mm/shmem.c +9
-4@@ -1584,13 +1584,13 @@ int shmem_unuse(unsigned int type)/** * shmem_writeout - Write the folio to swap +* @ctx: swap I/O context * @folio: The folio to write -* @plug: swap plug * @folio_list: list to put back folios on split * * Move the folio from the page cache to the swap cache. */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list) {struct address_space *mapping = folio->mapping;@@ -1702,7 +1702,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,shmem_delete_from_page_cache (folio,swp_to_radix_entry (folio->swap));BUG_ON (folio_mapped (folio)); - error =swap_writeout (folio, plug); + error =swap_writeout (ctx, folio);if (error !=AOP_WRITEPAGE_ACTIVATE ) {/* folio has been unlocked */ return error;@@ -1741,7 +1741,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,int shmem_write_folio (struct folio *folio) { -return shmem_writeout (folio,NULL ,NULL ); +struct swap_io_ctx ctx = {}; +int err; + + err =shmem_writeout (&ctx, folio,NULL ); +swap_write_submit (&ctx); +return err; }EXPORT_SYMBOL_GPL (shmem_write_folio);mm/swap.h +16
-20@@ -4,6 +4,7 @@#include <linux/atomic.h> /* for atomic_long_t */ #include <linux/mm.h> /* for PAGE_SHIFT */ +struct mempolicy ;struct swap_iocb ;struct swap_memcg_table ;@@ -78,6 +79,10 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; +struct swap_io_ctx { +struct swap_iocb *sio; +}; +#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ @@ -240,17 +245,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,/* linux/mm/page_io.c */ int sio_pool_init (void ); -struct swap_iocb ; -void swap_read_folio (struct folio *folio,struct swap_iocb **plug); -void __swap_read_unplug (struct swap_iocb *plug); -static inline void swap_read_unplug (struct swap_iocb *plug) -{ -if (unlikely (plug)) -__swap_read_unplug (plug); -} -void swap_write_unplug (struct swap_iocb *sio); -int swap_writeout (struct folio *folio,struct swap_iocb **swap_plug); -void __swap_writepage (struct folio *folio,struct swap_iocb **swap_plug); +void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio); +void swap_read_submit (struct swap_io_ctx *ctx); +void swap_write_submit (struct swap_io_ctx *ctx); +int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio); +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio);/* linux/mm/swap_state.c */ extern struct address_space swap_space__read_mostly ;@@ -317,9 +316,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,void show_swap_cache_info (void );void swapcache_clear (struct swap_info_struct *si,swp_entry_t entry,int nr); -struct folio *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask, -struct vm_area_struct *vma,unsigned long addr, -struct swap_iocb **plug); +struct folio *read_swap_cache_async (struct swap_io_ctx *ctx,swp_entry_t entry, +gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr);struct folio *swap_cluster_readahead (swp_entry_t entry,gfp_t flag,struct mempolicy *mpol,pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,@@ -335,7 +333,6 @@ static inline unsigned int folio_swap_flags(struct folio *folio) }#else /* CONFIG_SWAP */ -struct swap_iocb ;static inline struct swap_cluster_info *swap_cluster_lock (struct swap_info_struct *si,pgoff_t offset,bool irq) {@@ -381,11 +378,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } -static inline void swap_read_folio (struct folio *folio,struct swap_iocb **plug) +static inline void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) { } -static inline void swap_write_unplug (struct swap_iocb *sio) +static inline void swap_write_submit (struct swap_io_ctx *ctx) { }@@ -427,8 +424,7 @@ static inline void swap_update_readahead(struct folio *folio, { } -static inline int swap_writeout (struct folio *folio, -struct swap_iocb **swap_plug) +static inline int swap_writeout (struct swap_io_ctx *ctx,struct folio *folio) {return 0 ; }@@ -474,7 +470,7 @@ static inline unsigned int folio_swap_flags(struct folio *folio)#endif /* CONFIG_SWAP */ -int shmem_writeout (struct folio *folio,struct swap_iocb **plug, +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);#endif /* _MM_SWAP_H */ mm/swap_state.c +31
-22@@ -623,9 +623,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, } } -static struct folio *swap_cache_read_folio (swp_entry_t entry,gfp_t gfp, -struct mempolicy *mpol,pgoff_t ilx, -struct swap_iocb **plug,bool readahead) +static struct folio *swap_cache_read_folio (struct swap_io_ctx *ctx, +swp_entry_t entry,gfp_t gfp,struct mempolicy *mpol, +pgoff_t ilx,bool readahead) {struct folio *folio;@@ -639,7 +639,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,if (IS_ERR_OR_NULL (folio))return NULL ; -swap_read_folio (folio, plug); +swap_read_folio (ctx, folio);if (readahead) {folio_set_readahead (folio);count_vm_event (SWAP_RA );@@ -667,6 +667,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,struct folio *swapin_sync (swp_entry_t entry,gfp_t gfp,unsigned long orders,struct vm_fault *vmf,struct mempolicy *mpol,pgoff_t ilx) { +struct swap_io_ctx ctx = {};struct folio *folio; do {@@ -679,7 +680,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,if (IS_ERR (folio))return folio; -swap_read_folio (folio,NULL ); +swap_read_folio (&ctx, folio); +swap_read_submit (&ctx);return folio; }@@ -689,9 +691,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,* A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */ -struct folio *read_swap_cache_async (swp_entry_t entry,gfp_t gfp_mask, -struct vm_area_struct *vma,unsigned long addr, -struct swap_iocb **plug) +struct folio *read_swap_cache_async (struct swap_io_ctx *ctx,swp_entry_t entry, +gfp_t gfp_mask,struct vm_area_struct *vma,unsigned long addr) {struct swap_info_struct *si;struct mempolicy *mpol;@@ -703,13 +704,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,return NULL ; mpol =get_vma_policy (vma, addr,0 , &ilx); - folio =swap_cache_read_folio (entry, gfp_mask, mpol, ilx, plug, false); + folio =swap_cache_read_folio (ctx, entry, gfp_mask, mpol, ilx, false);mpol_cond_put (mpol);put_swap_device (si);return folio; } +static struct folio *swap_cache_read_folio_sync (swp_entry_t entry,gfp_t gfp, +struct mempolicy *mpol,pgoff_t ilx) +{ +struct swap_io_ctx ctx = {}; +struct folio *folio; + + folio =swap_cache_read_folio (&ctx, entry, gfp, mpol, ilx, false); +swap_read_submit (&ctx); +return folio; +} + static unsigned int __swapin_nr_pages(unsigned long prev_offset, unsigned long offset, int hits,@@ -798,8 +810,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,unsigned long start_offset, end_offset;unsigned long mask;struct swap_info_struct *si =__swap_entry_to_info (entry); +struct swap_io_ctx ctx = {};struct blk_plug plug; -struct swap_iocb *splug =NULL ;swp_entry_t ra_entry; mask =swapin_nr_pages (offset) -1 ;@@ -818,18 +830,17 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,for (offset = start_offset; offset <= end_offset ; offset++) {/* Ok, do the async read-ahead now */ ra_entry =swp_entry (swp_type (entry), offset); - folio =swap_cache_read_folio (ra_entry, gfp_mask, mpol, ilx, - &splug, offset != entry_offset); + folio =swap_cache_read_folio (&ctx, ra_entry, gfp_mask, mpol, + ilx, offset != entry_offset);if (!folio)continue ;folio_put (folio); }blk_finish_plug (&plug); -swap_read_unplug (splug); +swap_read_submit (&ctx);lru_add_drain ();/* Push any new pages onto the LRU now */ skip: -/* The page was likely read above, so no need for plugging here */ -return swap_cache_read_folio (entry, gfp_mask, mpol, ilx,NULL , false); +return swap_cache_read_folio_sync (entry, gfp_mask, mpol, ilx); } static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,@@ -889,8 +900,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,static struct folio *swap_vma_readahead (swp_entry_t targ_entry,gfp_t gfp_mask,struct mempolicy *mpol,pgoff_t targ_ilx,struct vm_fault *vmf) { +struct swap_io_ctx ctx = {};struct blk_plug plug; -struct swap_iocb *splug =NULL ;struct folio *folio;pte_t *pte =NULL , pentry;int win;@@ -929,8 +940,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,if (!si)continue ; } - folio =swap_cache_read_folio (entry, gfp_mask, mpol, ilx, - &splug, addr != vmf->address); + folio =swap_cache_read_folio (&ctx, entry, gfp_mask, mpol, ilx, + addr != vmf->address);if (si)put_swap_device (si);if (!folio)@@ -940,13 +951,11 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,if (pte)pte_unmap (pte);blk_finish_plug (&plug); -swap_read_unplug (splug); +swap_read_submit (&ctx);lru_add_drain (); skip:/* The folio was likely read above, so no need for plugging here */ - folio =swap_cache_read_folio (targ_entry, gfp_mask, mpol, targ_ilx, -NULL , false); -return folio; +return swap_cache_read_folio_sync (targ_entry, gfp_mask, mpol, targ_ilx); }/** mm/vmscan.c +7
-8@@ -615,8 +615,8 @@ typedef enum {/* * pageout is called by shrink_folio_list() for each dirty folio. */ -static pageout_t pageout (struct folio *folio,struct address_space *mapping, -struct swap_iocb **plug,struct list_head *folio_list) +static pageout_t pageout (struct swap_io_ctx *ctx,struct address_space *mapping, +struct folio *folio,struct list_head *folio_list) {int res;@@ -652,9 +652,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,* the split out folios get added back to folio_list. */ if (shmem_mapping (mapping)) - res =shmem_writeout (folio, plug, folio_list); + res =shmem_writeout (ctx, folio, folio_list);else - res =swap_writeout (folio, plug); + res =swap_writeout (ctx, folio);if (res <0 )handle_write_error (mapping, folio, res);@@ -1063,7 +1063,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,unsigned int nr_reclaimed =0 , nr_demoted =0 ;unsigned int pgactivate =0 ;bool do_demote_pass; -struct swap_iocb *plug =NULL ; +struct swap_io_ctx ctx = {};folio_batch_init (&free_folios);memset (stat,0 ,sizeof (*stat));@@ -1394,7 +1394,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,* starts and then write it out here. */ try_to_unmap_flush_dirty (); -switch (pageout (folio, mapping, &plug, folio_list)) { +switch (pageout (&ctx, mapping, folio, folio_list)) {case PAGE_KEEP : goto keep_locked;case PAGE_ACTIVATE :@@ -1584,8 +1584,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,list_splice (&ret_folios, folio_list);count_vm_events (PGACTIVATE , pgactivate); -if (plug) -swap_write_unplug (plug); +swap_write_submit (&ctx);return nr_reclaimed; }mm/zswap.c +1
-0@@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,struct folio *folio;struct mempolicy *mpol;struct swap_info_struct *si; +struct swap_io_ctx ctx = {};int ret =0 ;/* try to allocate swap cache folio */
mm/zswap.c static int zswap_writeback_entry(struct zswap_entry *entry, · line 1054
Quoted 6 lines
mm/zswap.c +2
-1@@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,folio_set_reclaim (folio);/* start writeback */ -__swap_writepage (folio,NULL ); +__swap_writepage (&ctx, folio); +swap_write_submit (&ctx);
LGTM.
Note to self and other zswap folks: explore batching IO for zswap writeback. Should be fairly simple to maintain swap_io_ctx for each writeback round and pass it around. Not sure how much physical swap address space contiguity these zswap entries still have at the point of zswap writeback, as zswap-ins might have already punched holes. But can't be worse than status quo, and soon we will able to defer the allocation of physical swap space until writeback time with swap virtualization, which will make batching more attractive here :)
Re: [PATCH 3/8] mm/swap: introduce struct swap_io_ctx
On Fri, Jun 05, 2026 at 10:41:56AM -0700, Nhat Pham wrote:
[fullquote]
Can you trim the parts of the mail that you are not replying to? You've quoted the entire mail without any comment on the patch, just to add a note at the end.
Quoted 3 lines
Note to self and other zswap folks: explore batching IO for zswap writeback. Should be fairly simple to maintain
swap_io_ctxfor each writeback round and pass it around.
Yes, compression should benefit from larger chunks to work on. I think the same would also apply to zram (to respond to Chris' comment elsewhere).
PATCH 4/8 mm/swap: also use struct swap_iocb for block I/O
In reply to Christoph HellwigBlock I/O benefits from batching just as much as remote file systems. Extent struct swap_iocb to support building a bio on the fly as well, and rewrite the block based swap code for it. This especially benefits submit_bio based drivers that do not have the block plugging available, but also saves allocating extra bios for blk-mq drivers.
Note that the block based swap code now uses the same memcg-based check previously added for file system based swap as well.
--- mm/page_io.c | 526 ++++++++++++++++++++++++-------------------------- mm/swap.h | 1 + mm/swapfile.c | 9 +- 3 files changed, 252 insertions(+), 284 deletions(-)
mm/page_io.c +247 -279
@@ -28,54 +28,6 @@#include "swap.h" #include "swap_table.h" -static void __end_swap_bio_write (struct bio *bio) -{ -struct folio *folio =bio_first_folio_all (bio); - -if (bio->bi_status) { -/* -* We failed to write the page out to swap-space. -* Re-dirty the page in order to avoid it being reclaimed. -* Also print a dire warning that things will go BAD (tm) -* very quickly. -* -* Also clear PG_reclaim to avoid folio_rotate_reclaimable() -*/ -folio_mark_dirty (folio); -pr_alert_ratelimited ("Write-error on swap-device (%u:%u:%llu)\n" , -MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), - (unsigned long long )bio->bi_iter.bi_sector); -folio_clear_reclaim (folio); - } -folio_end_writeback (folio); -} - -static void end_swap_bio_write (struct bio *bio) -{ -__end_swap_bio_write (bio); -bio_put (bio); -} - -static void __end_swap_bio_read (struct bio *bio) -{ -struct folio *folio =bio_first_folio_all (bio); - -if (bio->bi_status) { -pr_alert_ratelimited ("Read-error on swap-device (%u:%u:%llu)\n" , -MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), - (unsigned long long )bio->bi_iter.bi_sector); - }else { -folio_mark_uptodate (folio); - } -folio_unlock (folio); -} - -static void end_swap_bio_read (struct bio *bio) -{ -__end_swap_bio_read (bio); -bio_put (bio); -} -int generic_swapfile_activate (struct swap_info_struct *sis,struct file *swap_file,sector_t *span)
@@ -316,26 +268,47 @@ static inline void count_swpout_vm_event(struct folio *folio) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -static void bio_associate_blkg_from_page (struct bio *bio,struct folio *folio) +static struct cgroup_subsys_state *folio_memcg_blkg_css (struct folio *folio) +{ +return cgroup_e_css (folio_memcg (folio)->css.cgroup, &io_cgrp_subsys); +} + +static bool folio_blkg_can_merge (struct folio *folio,struct folio *prev_folio) { -struct cgroup_subsys_state *css; -struct mem_cgroup *memcg; +if (!folio_memcg_charged (folio) || !folio_memcg_charged (prev_folio)) +return true; + +rcu_read_lock (); +if (folio_memcg_blkg_css (folio) !=folio_memcg_blkg_css (prev_folio)) { +rcu_read_unlock (); +return false; + } +rcu_read_unlock (); + +return true; +} +static void bio_associate_blkg_from_page (struct bio *bio,struct folio *folio) +{if (!folio_memcg_charged (folio))return ; -rcu_read_lock (); - memcg =folio_memcg (folio); - css =cgroup_e_css (memcg->css.cgroup, &io_cgrp_subsys); -bio_associate_blkg_from_css (bio, css); +bio_associate_blkg_from_css (bio,folio_memcg_blkg_css (folio));rcu_read_unlock (); }#else +static bool folio_blkg_can_merge (struct folio *folio,struct folio *prev_folio) +{ +return true; +}#define bio_associate_blkg_from_page (bio, folio) do { } while (0)#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ struct swap_iocb { -struct kiocb iocb; +union { +struct kiocb iocb; +struct bio bio; + };struct bio_vec bvecs[SWAP_CLUSTER_MAX ];int nr_bvecs;int len;
@@ -355,171 +328,70 @@ int sio_pool_init(void)return 0 ; } -static void sio_write_complete (struct kiocb *iocb,long ret) +static bool swap_can_merge (struct swap_io_ctx *ctx,struct folio *folio, +int rw) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); -struct page *page = sio->bvecs[0 ].bv_page; -int p; +struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs -1 ]; +struct folio *prev_folio =page_folio (last_bv->bv_page); +size_t prev_folio_size =folio_size (prev_folio); -if (ret != sio->len) { -/* -* In the case of swap-over-nfs, this can be a -* temporary failure if the system has limited -* memory for allocating transmit buffers. -* Mark the page dirty and avoid -* folio_rotate_reclaimable but rate-limit the -* messages. -*/ -pr_err_ratelimited ("Write error %ld on dio swapfile (%llu)\n" , - ret,swap_dev_pos (page_swap_entry (page))); -for (p =0 ; p < sio->nr_bvecs; p++) { - page = sio->bvecs[p].bv_page; -set_page_dirty (page); -ClearPageReclaim (page); - } - } +if (ctx->sis != sis) +return false; -for (p =0 ; p < sio->nr_bvecs; p++) -end_page_writeback (sio->bvecs[p].bv_page); +if (sis->flags &SWP_FS_OPS ) { +if (swap_dev_pos (folio->swap) != +swap_dev_pos (prev_folio->swap) + prev_folio_size) +return false; + }else { +if (swap_folio_sector (folio) != +swap_folio_sector (prev_folio) + + (prev_folio_size >>SECTOR_SHIFT )) +return false; +if (rw ==WRITE && !folio_blkg_can_merge (folio, prev_folio)) +return false; + } -mempool_free (sio, sio_pool); +return true; } -static void swap_writepage_fs (struct swap_io_ctx *ctx,struct folio *folio) +static void swap_add_page (struct swap_io_ctx *ctx,struct folio *folio,int rw) { -struct swap_iocb *sio = ctx->sio;struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct file *swap_file = sis->swap_file; -loff_t pos =swap_dev_pos (folio->swap); +struct swap_iocb *sio = ctx->sio; -count_swpout_vm_event (folio); -folio_start_writeback (folio); -folio_unlock (folio); -if (sio) { -if (sio->iocb.ki_filp != swap_file || - sio->iocb.ki_pos + sio->len != pos) { +if (sio && !swap_can_merge (ctx, folio, rw)) { +if (rw ==WRITE )swap_write_submit (ctx); - sio =NULL ; - } +else +swap_read_submit (ctx); + sio = ctx->sio; } +if (!sio) { - sio =mempool_alloc (sio_pool,GFP_NOIO ); -init_sync_kiocb (&sio->iocb, swap_file); - sio->iocb.ki_complete = sio_write_complete; - sio->iocb.ki_pos = pos; + ctx->sis = sis; + ctx->sio = sio =mempool_alloc (sio_pool,GFP_NOIO ); sio->nr_bvecs =0 ; sio->len =0 ; }bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); - sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { -swap_write_submit (ctx); - sio =NULL ; +if (++sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +if (rw ==WRITE ) +swap_write_submit (ctx); +else +swap_read_submit (ctx); } - ctx->sio = sio; -} - -static void swap_writepage_bdev_sync (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio_vec bv; -struct bio bio; - -bio_init (&bio, sis->bdev, &bv,1 ,REQ_OP_WRITE |REQ_SWAP ); - bio.bi_iter.bi_sector =swap_folio_sector (folio); -bio_add_folio_nofail (&bio, folio,folio_size (folio),0 ); - -bio_associate_blkg_from_page (&bio, folio); -count_swpout_vm_event (folio); - -folio_start_writeback (folio); -folio_unlock (folio); - -submit_bio_wait (&bio); -__end_swap_bio_write (&bio); } -static void swap_writepage_bdev_async (struct folio *folio, -struct swap_info_struct *sis) +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) { -struct bio *bio; - - bio =bio_alloc (sis->bdev,1 ,REQ_OP_WRITE |REQ_SWAP ,GFP_NOIO ); - bio->bi_iter.bi_sector =swap_folio_sector (folio); - bio->bi_end_io = end_swap_bio_write; -bio_add_folio_nofail (bio, folio,folio_size (folio),0 ); +VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -bio_associate_blkg_from_page (bio, folio);count_swpout_vm_event (folio);folio_start_writeback (folio);folio_unlock (folio); -submit_bio (bio); -} - -void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) -{ -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); - -VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -/* -* ->flags can be updated non-atomically, -* but that will never affect SWP_FS_OPS, so the data_race -* is safe. -*/ -if (data_race (sis->flags &SWP_FS_OPS )) -swap_writepage_fs (ctx, folio); -/* -* ->flags can be updated non-atomically, -* but that will never affect SWP_SYNCHRONOUS_IO, so the data_race -* is safe. -*/ -else if (data_race (sis->flags &SWP_SYNCHRONOUS_IO )) -swap_writepage_bdev_sync (folio, sis); -else -swap_writepage_bdev_async (folio, sis); -} - -void swap_write_submit (struct swap_io_ctx *ctx) -{ -struct swap_iocb *sio = ctx->sio; -struct iov_iter from; -int ret; - -if (!sio) -return ; - -iov_iter_bvec (&from,ITER_SOURCE , sio->bvecs, sio->nr_bvecs, sio->len); - ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from); -if (ret != -EIOCBQUEUED ) -sio_write_complete (&sio->iocb, ret); - ctx->sio =NULL ; -} - -static void sio_read_complete (struct kiocb *iocb,long ret) -{ - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); -int p; - -if (ret == sio->len) { -for (p =0 ; p < sio->nr_bvecs; p++) { -struct folio *folio =page_folio (sio->bvecs[p].bv_page); - -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -folio_mark_uptodate (folio); -folio_unlock (folio); - } -count_vm_events (PSWPIN , sio->len >>PAGE_SHIFT ); - }else { -for (p =0 ; p < sio->nr_bvecs; p++) { -struct folio *folio =page_folio (sio->bvecs[p].bv_page); - -folio_unlock (folio); - } -pr_alert_ratelimited ("Read-error on swap-device\n" ); - } -mempool_free (sio, sio_pool); +swap_add_page (ctx, folio,WRITE ); }/*
@@ -585,74 +457,6 @@ static bool swap_read_folio_zeromap(struct folio *folio)return true; } -static void swap_read_folio_fs (struct swap_io_ctx *ctx,struct folio *folio) -{ -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct swap_iocb *sio = ctx->sio; -loff_t pos =swap_dev_pos (folio->swap); - -if (sio) { -if (sio->iocb.ki_filp != sis->swap_file || - sio->iocb.ki_pos + sio->len != pos) { -swap_read_submit (ctx); - sio =NULL ; - } - } -if (!sio) { - sio =mempool_alloc (sio_pool,GFP_KERNEL ); -init_sync_kiocb (&sio->iocb, sis->swap_file); - sio->iocb.ki_pos = pos; - sio->iocb.ki_complete = sio_read_complete; - sio->nr_bvecs =0 ; - sio->len =0 ; - } -bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); - sio->len +=folio_size (folio); - sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { -swap_read_submit (ctx); - sio =NULL ; - } - ctx->sio = sio; -} - -static void swap_read_folio_bdev_sync (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio_vec bv; -struct bio bio; - -bio_init (&bio, sis->bdev, &bv,1 ,REQ_OP_READ ); - bio.bi_iter.bi_sector =swap_folio_sector (folio); -bio_add_folio_nofail (&bio, folio,folio_size (folio),0 ); -/* -* Keep this task valid during swap readpage because the oom killer may -* attempt to access it in the page fault retry time check. -*/ -get_task_struct (current); -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -count_vm_events (PSWPIN ,folio_nr_pages (folio)); -submit_bio_wait (&bio); -__end_swap_bio_read (&bio); -put_task_struct (current); -} - -static void swap_read_folio_bdev_async (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio *bio; - - bio =bio_alloc (sis->bdev,1 ,REQ_OP_READ ,GFP_KERNEL ); - bio->bi_iter.bi_sector =swap_folio_sector (folio); - bio->bi_end_io = end_swap_bio_read; -bio_add_folio_nofail (bio, folio,folio_size (folio),0 ); -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -count_vm_events (PSWPIN ,folio_nr_pages (folio)); -submit_bio (bio); -} -void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);
@@ -686,14 +490,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)/* We have to read from slower devices. Increase zswap protection. */ zswap_folio_swapin (folio); - -if (data_race (sis->flags &SWP_FS_OPS )) { -swap_read_folio_fs (ctx, folio); - }else if (synchronous) { -swap_read_folio_bdev_sync (folio, sis); - }else { -swap_read_folio_bdev_async (folio, sis); - } +swap_add_page (ctx, folio,READ ); finish:if (workingset) {
@@ -703,18 +500,189 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)delayacct_swapin_end (); } -void swap_read_submit (struct swap_io_ctx *ctx) +static void swap_write_end (struct swap_iocb *sio,bool failed) +{ +int p; + +for (p =0 ; p < sio->nr_bvecs; p++) { +struct page *page = sio->bvecs[p].bv_page; + +if (failed) { +set_page_dirty (page); +ClearPageReclaim (page); + } +end_page_writeback (page); + } +mempool_free (sio, sio_pool); +} + +static void swap_fs_write_complete (struct kiocb *iocb,long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); +bool failed = ret != sio->len; + +if (failed) { +struct page *page = sio->bvecs[0 ].bv_page; + +/* +* In the case of swap-over-nfs, this can be a temporary failure +* if the system has limited memory for allocating transmit +* buffers. Mark the page dirty and avoid +* folio_rotate_reclaimable but rate-limit the messages. +*/ +pr_err_ratelimited ("Write error %ld on dio swapfile (%llu)\n" , + ret,swap_dev_pos (page_swap_entry (page))); + } + +swap_write_end (sio, failed); +} + +static void end_swap_bio_write (struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); +bool failed = !!bio->bi_status; + +if (failed) +pr_alert_ratelimited ("Write-error on swap-device (%u:%u:%llu)\n" , +MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), + (unsigned long long )bio->bi_iter.bi_sector); +bio_uninit (bio); +swap_write_end (sio, failed); +} + +static void swap_read_end (struct swap_iocb *sio,bool failed) +{ +int p; + +for (p =0 ; p < sio->nr_bvecs; p++) { +struct folio *folio =page_folio (sio->bvecs[p].bv_page); + +if (!failed) { +count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); +count_memcg_folio_events (folio,PSWPIN , +folio_nr_pages (folio)); +folio_mark_uptodate (folio); + } +folio_unlock (folio); + } + +if (!failed) +count_vm_events (PSWPIN , sio->len >>PAGE_SHIFT ); + +mempool_free (sio, sio_pool); +} + +static void swap_fs_read_complete (struct kiocb *iocb,long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); +bool failed = ret != sio->len; + +if (failed) +pr_alert_ratelimited ("Read-error on swap-device\n" ); +swap_read_end (sio, failed); +} + +static void swap_bio_read_end_io (struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); +bool failed = !!bio->bi_status; + +if (failed) +pr_alert_ratelimited ("Read-error on swap-device (%u:%u:%llu)\n" , +MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), + (unsigned long long )bio->bi_iter.bi_sector); +bio_uninit (bio); +swap_read_end (sio, failed); +} + +static void swap_bdev_submit_write (struct swap_io_ctx *ctx) {struct swap_iocb *sio = ctx->sio; -struct iov_iter from; +struct bio *bio = &sio->bio; + +bio_init (bio, ctx->sis->bdev, sio->bvecs,ARRAY_SIZE (sio->bvecs), +REQ_OP_WRITE |REQ_SWAP ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector =swap_folio_sector (bio_first_folio_all (bio)); +bio_associate_blkg_from_page (bio,bio_first_folio_all (bio)); + +if (ctx->sis->flags &SWP_SYNCHRONOUS_IO ) { +submit_bio_wait (bio); +end_swap_bio_write (bio); + }else { + bio->bi_end_io = end_swap_bio_write; +submit_bio (bio); + } +} + +static void swap_bdev_submit_read (struct swap_io_ctx *ctx) +{ +struct swap_iocb *sio = ctx->sio; +struct bio *bio = &sio->bio; + +bio_init (bio, ctx->sis->bdev, sio->bvecs,ARRAY_SIZE (sio->bvecs), +REQ_OP_READ ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector =swap_folio_sector (bio_first_folio_all (bio)); + +if (ctx->sis->flags &SWP_SYNCHRONOUS_IO ) { +/* +* Keep this task valid during swap readpage because the oom +* killer may attempt to access it in the page fault retry +* time check. +*/ +get_task_struct (current); +submit_bio_wait (bio); +swap_bio_read_end_io (bio); +put_task_struct (current); + }else { + bio->bi_end_io = swap_bio_read_end_io; +submit_bio (bio); + } +} + +static void swap_fs_submit (struct swap_io_ctx *ctx,int rw) +{ +struct swap_iocb *sio = ctx->sio; +struct iov_iter iter;int ret; -if (!sio) -return ; +init_sync_kiocb (&sio->iocb, ctx->sis->swap_file); + sio->iocb.ki_pos =swap_dev_pos (page_folio (sio->bvecs[0 ].bv_page)->swap); +if (rw ==WRITE ) + sio->iocb.ki_complete = swap_fs_write_complete; +else + sio->iocb.ki_complete = swap_fs_read_complete; -iov_iter_bvec (&from,ITER_DEST , sio->bvecs, sio->nr_bvecs, sio->len); - ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from); +iov_iter_bvec (&iter, rw ==WRITE ?ITER_SOURCE :ITER_DEST , + sio->bvecs, sio->nr_bvecs, sio->len); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &iter);if (ret != -EIOCBQUEUED ) -sio_read_complete (&sio->iocb, ret); + sio->iocb.ki_complete (&sio->iocb, ret); +} + +void swap_write_submit (struct swap_io_ctx *ctx) +{ +if (!ctx->sio) +return ; + +if (ctx->sis->flags &SWP_FS_OPS ) +swap_fs_submit (ctx,WRITE ); +else +swap_bdev_submit_write (ctx); + ctx->sio =NULL ; + ctx->sis =NULL ; +} + +void swap_read_submit (struct swap_io_ctx *ctx) +{ +if (!ctx->sio) +return ; + +if (ctx->sis->flags &SWP_FS_OPS ) +swap_fs_submit (ctx,READ ); +else +swap_bdev_submit_read (ctx); ctx->sio =NULL ; + ctx->sis =NULL ; }
mm/swap.h +1 -0
@@ -81,6 +81,7 @@ enum swap_cluster_flags {struct swap_io_ctx {struct swap_iocb *sio; +struct swap_info_struct *sis; };#ifdef CONFIG_SWAP
mm/swapfile.c +4 -5
@@ -2842,6 +2842,10 @@ static int setup_swap_extents(struct swap_info_struct *sis,struct inode *inode = mapping->host;int ret; + ret =sio_pool_init (); +if (ret) +return ret; +if (S_ISBLK (inode->i_mode)) { ret =add_swap_extent (sis,0 , sis->max,0 ); *span = sis->pages;
@@ -2853,11 +2857,6 @@ static int setup_swap_extents(struct swap_info_struct *sis,if (ret <0 )return ret; sis->flags |=SWP_ACTIVATED ; -if ((sis->flags &SWP_FS_OPS ) && -sio_pool_init () !=0 ) { -destroy_swap_extents (sis, swap_file); -return -ENOMEM ; - }return ret; }
Signature · 1 line
2.53.0
Baoquan He
reviewerRe: [PATCH 4/8] mm/swap: also use struct swap_iocb for block I/O
Quoted 2 lines · Christoph Hellwig
Block I/O benefits from batching just as much as remote file systems. Extent struct
swap_iocbto support building a bio on the fly as well,
~~~ Extend, typo
Quoted 14 lines
and rewrite the block based swap code for it. This especially benefits
submit_biobased drivers that do not have the block plugging available, but also saves allocating extra bios for blk-mq drivers.Note that the block based swap code now uses the same memcg-based check previously added for file system based swap as well.
Signed-off-by Christoph Hellwig <[email protected]>--- mm/page_io.c | 526 ++++++++++++++++++++++++-------------------------- mm/swap.h | 1 + mm/swapfile.c | 9 +- 3 files changed, 252 insertions(+), 284 deletions(-)
...snip...
Baoquan He
reviewerRe: [PATCH 4/8] mm/swap: also use struct swap_iocb for block I/O
In reply to Christoph HellwigQuoted 8 lines · Christoph Hellwig
Block I/O benefits from batching just as much as remote file systems. Extent struct
swap_iocbto support building a bio on the fly as well, and rewrite the block based swap code for it. This especially benefitssubmit_biobased drivers that do not have the block plugging available, but also saves allocating extra bios for blk-mq drivers.Note that the block based swap code now uses the same memcg-based check previously added for file system based swap as well.
I would add below words in log to ease patch reviewing/studying, while it could be personal style.
What it does:
- Adds bio to a union with kiocb in struct swap_iocb, so the same sio can drive either FS or block I/O;
- Removes 6 functions (swap_writepage_bdev_sync, swap_writepage_bdev_async, swap_read_folio_bdev_sync, swap_read_folio_bdev_async, swap_writepage_fs, swap_read_folio_fs) and replaces them with a unified (swap_add_page + swap_can_merge + submit path);
- Adds sis pointer to swap_io_ctx for direct access in the submit path
- sio_pool_init() is now called unconditionally (not just for FS swap), which is necessary since bdev swap now also allocates from the sio pool;
swap_add_page() should be rename to swap_add_folio();
Quoted 664 lines
Signed-off-by Christoph Hellwig <[email protected]>--- mm/page_io.c | 526 ++++++++++++++++++++++++-------------------------- mm/swap.h | 1 + mm/swapfile.c | 9 +- 3 files changed, 252 insertions(+), 284 deletions(-)mm/page_io.c +247
-279@@ -28,54 +28,6 @@#include "swap.h" #include "swap_table.h" -static void __end_swap_bio_write (struct bio *bio) -{ -struct folio *folio =bio_first_folio_all (bio); - -if (bio->bi_status) { -/* -* We failed to write the page out to swap-space. -* Re-dirty the page in order to avoid it being reclaimed. -* Also print a dire warning that things will go BAD (tm) -* very quickly. -* -* Also clear PG_reclaim to avoid folio_rotate_reclaimable() -*/ -folio_mark_dirty (folio); -pr_alert_ratelimited ("Write-error on swap-device (%u:%u:%llu)\n" , -MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), - (unsigned long long )bio->bi_iter.bi_sector); -folio_clear_reclaim (folio); - } -folio_end_writeback (folio); -} - -static void end_swap_bio_write (struct bio *bio) -{ -__end_swap_bio_write (bio); -bio_put (bio); -} - -static void __end_swap_bio_read (struct bio *bio) -{ -struct folio *folio =bio_first_folio_all (bio); - -if (bio->bi_status) { -pr_alert_ratelimited ("Read-error on swap-device (%u:%u:%llu)\n" , -MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), - (unsigned long long )bio->bi_iter.bi_sector); - }else { -folio_mark_uptodate (folio); - } -folio_unlock (folio); -} - -static void end_swap_bio_read (struct bio *bio) -{ -__end_swap_bio_read (bio); -bio_put (bio); -} -int generic_swapfile_activate (struct swap_info_struct *sis,struct file *swap_file,sector_t *span)@@ -316,26 +268,47 @@ static inline void count_swpout_vm_event(struct folio *folio) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -static void bio_associate_blkg_from_page (struct bio *bio,struct folio *folio) +static struct cgroup_subsys_state *folio_memcg_blkg_css (struct folio *folio) +{ +return cgroup_e_css (folio_memcg (folio)->css.cgroup, &io_cgrp_subsys); +} + +static bool folio_blkg_can_merge (struct folio *folio,struct folio *prev_folio) { -struct cgroup_subsys_state *css; -struct mem_cgroup *memcg; +if (!folio_memcg_charged (folio) || !folio_memcg_charged (prev_folio)) +return true; + +rcu_read_lock (); +if (folio_memcg_blkg_css (folio) !=folio_memcg_blkg_css (prev_folio)) { +rcu_read_unlock (); +return false; + } +rcu_read_unlock (); + +return true; +} +static void bio_associate_blkg_from_page (struct bio *bio,struct folio *folio) +{if (!folio_memcg_charged (folio))return ; -rcu_read_lock (); - memcg =folio_memcg (folio); - css =cgroup_e_css (memcg->css.cgroup, &io_cgrp_subsys); -bio_associate_blkg_from_css (bio, css); +bio_associate_blkg_from_css (bio,folio_memcg_blkg_css (folio));rcu_read_unlock (); }#else +static bool folio_blkg_can_merge (struct folio *folio,struct folio *prev_folio) +{ +return true; +}#define bio_associate_blkg_from_page (bio, folio) do { } while (0)#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ struct swap_iocb { -struct kiocb iocb; +union { +struct kiocb iocb; +struct bio bio; + };struct bio_vec bvecs[SWAP_CLUSTER_MAX ];int nr_bvecs;int len;@@ -355,171 +328,70 @@ int sio_pool_init(void)return 0 ; } -static void sio_write_complete (struct kiocb *iocb,long ret) +static bool swap_can_merge (struct swap_io_ctx *ctx,struct folio *folio, +int rw) { - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); -struct page *page = sio->bvecs[0 ].bv_page; -int p; +struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs -1 ]; +struct folio *prev_folio =page_folio (last_bv->bv_page); +size_t prev_folio_size =folio_size (prev_folio); -if (ret != sio->len) { -/* -* In the case of swap-over-nfs, this can be a -* temporary failure if the system has limited -* memory for allocating transmit buffers. -* Mark the page dirty and avoid -* folio_rotate_reclaimable but rate-limit the -* messages. -*/ -pr_err_ratelimited ("Write error %ld on dio swapfile (%llu)\n" , - ret,swap_dev_pos (page_swap_entry (page))); -for (p =0 ; p < sio->nr_bvecs; p++) { - page = sio->bvecs[p].bv_page; -set_page_dirty (page); -ClearPageReclaim (page); - } - } +if (ctx->sis != sis) +return false; -for (p =0 ; p < sio->nr_bvecs; p++) -end_page_writeback (sio->bvecs[p].bv_page); +if (sis->flags &SWP_FS_OPS ) { +if (swap_dev_pos (folio->swap) != +swap_dev_pos (prev_folio->swap) + prev_folio_size) +return false; + }else { +if (swap_folio_sector (folio) != +swap_folio_sector (prev_folio) + + (prev_folio_size >>SECTOR_SHIFT )) +return false; +if (rw ==WRITE && !folio_blkg_can_merge (folio, prev_folio)) +return false; + } -mempool_free (sio, sio_pool); +return true; } -static void swap_writepage_fs (struct swap_io_ctx *ctx,struct folio *folio) +static void swap_add_page (struct swap_io_ctx *ctx,struct folio *folio,int rw) { -struct swap_iocb *sio = ctx->sio;struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct file *swap_file = sis->swap_file; -loff_t pos =swap_dev_pos (folio->swap); +struct swap_iocb *sio = ctx->sio; -count_swpout_vm_event (folio); -folio_start_writeback (folio); -folio_unlock (folio); -if (sio) { -if (sio->iocb.ki_filp != swap_file || - sio->iocb.ki_pos + sio->len != pos) { +if (sio && !swap_can_merge (ctx, folio, rw)) { +if (rw ==WRITE )swap_write_submit (ctx); - sio =NULL ; - } +else +swap_read_submit (ctx); + sio = ctx->sio; } +if (!sio) { - sio =mempool_alloc (sio_pool,GFP_NOIO ); -init_sync_kiocb (&sio->iocb, swap_file); - sio->iocb.ki_complete = sio_write_complete; - sio->iocb.ki_pos = pos; + ctx->sis = sis; + ctx->sio = sio =mempool_alloc (sio_pool,GFP_NOIO ); sio->nr_bvecs =0 ; sio->len =0 ; }bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); - sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { -swap_write_submit (ctx); - sio =NULL ; +if (++sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { +if (rw ==WRITE ) +swap_write_submit (ctx); +else +swap_read_submit (ctx); } - ctx->sio = sio; -} - -static void swap_writepage_bdev_sync (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio_vec bv; -struct bio bio; - -bio_init (&bio, sis->bdev, &bv,1 ,REQ_OP_WRITE |REQ_SWAP ); - bio.bi_iter.bi_sector =swap_folio_sector (folio); -bio_add_folio_nofail (&bio, folio,folio_size (folio),0 ); - -bio_associate_blkg_from_page (&bio, folio); -count_swpout_vm_event (folio); - -folio_start_writeback (folio); -folio_unlock (folio); - -submit_bio_wait (&bio); -__end_swap_bio_write (&bio); } -static void swap_writepage_bdev_async (struct folio *folio, -struct swap_info_struct *sis) +void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) { -struct bio *bio; - - bio =bio_alloc (sis->bdev,1 ,REQ_OP_WRITE |REQ_SWAP ,GFP_NOIO ); - bio->bi_iter.bi_sector =swap_folio_sector (folio); - bio->bi_end_io = end_swap_bio_write; -bio_add_folio_nofail (bio, folio,folio_size (folio),0 ); +VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -bio_associate_blkg_from_page (bio, folio);count_swpout_vm_event (folio);folio_start_writeback (folio);folio_unlock (folio); -submit_bio (bio); -} - -void __swap_writepage (struct swap_io_ctx *ctx,struct folio *folio) -{ -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); - -VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -/* -* ->flags can be updated non-atomically, -* but that will never affect SWP_FS_OPS, so the data_race -* is safe. -*/ -if (data_race (sis->flags &SWP_FS_OPS )) -swap_writepage_fs (ctx, folio); -/* -* ->flags can be updated non-atomically, -* but that will never affect SWP_SYNCHRONOUS_IO, so the data_race -* is safe. -*/ -else if (data_race (sis->flags &SWP_SYNCHRONOUS_IO )) -swap_writepage_bdev_sync (folio, sis); -else -swap_writepage_bdev_async (folio, sis); -} - -void swap_write_submit (struct swap_io_ctx *ctx) -{ -struct swap_iocb *sio = ctx->sio; -struct iov_iter from; -int ret; - -if (!sio) -return ; - -iov_iter_bvec (&from,ITER_SOURCE , sio->bvecs, sio->nr_bvecs, sio->len); - ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from); -if (ret != -EIOCBQUEUED ) -sio_write_complete (&sio->iocb, ret); - ctx->sio =NULL ; -} - -static void sio_read_complete (struct kiocb *iocb,long ret) -{ - struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); -int p; - -if (ret == sio->len) { -for (p =0 ; p < sio->nr_bvecs; p++) { -struct folio *folio =page_folio (sio->bvecs[p].bv_page); - -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -folio_mark_uptodate (folio); -folio_unlock (folio); - } -count_vm_events (PSWPIN , sio->len >>PAGE_SHIFT ); - }else { -for (p =0 ; p < sio->nr_bvecs; p++) { -struct folio *folio =page_folio (sio->bvecs[p].bv_page); - -folio_unlock (folio); - } -pr_alert_ratelimited ("Read-error on swap-device\n" ); - } -mempool_free (sio, sio_pool); +swap_add_page (ctx, folio,WRITE ); }/* @@ -585,74 +457,6 @@ static bool swap_read_folio_zeromap(struct folio *folio)return true; } -static void swap_read_folio_fs (struct swap_io_ctx *ctx,struct folio *folio) -{ -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); -struct swap_iocb *sio = ctx->sio; -loff_t pos =swap_dev_pos (folio->swap); - -if (sio) { -if (sio->iocb.ki_filp != sis->swap_file || - sio->iocb.ki_pos + sio->len != pos) { -swap_read_submit (ctx); - sio =NULL ; - } - } -if (!sio) { - sio =mempool_alloc (sio_pool,GFP_KERNEL ); -init_sync_kiocb (&sio->iocb, sis->swap_file); - sio->iocb.ki_pos = pos; - sio->iocb.ki_complete = sio_read_complete; - sio->nr_bvecs =0 ; - sio->len =0 ; - } -bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); - sio->len +=folio_size (folio); - sio->nr_bvecs +=1 ; -if (sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { -swap_read_submit (ctx); - sio =NULL ; - } - ctx->sio = sio; -} - -static void swap_read_folio_bdev_sync (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio_vec bv; -struct bio bio; - -bio_init (&bio, sis->bdev, &bv,1 ,REQ_OP_READ ); - bio.bi_iter.bi_sector =swap_folio_sector (folio); -bio_add_folio_nofail (&bio, folio,folio_size (folio),0 ); -/* -* Keep this task valid during swap readpage because the oom killer may -* attempt to access it in the page fault retry time check. -*/ -get_task_struct (current); -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -count_vm_events (PSWPIN ,folio_nr_pages (folio)); -submit_bio_wait (&bio); -__end_swap_bio_read (&bio); -put_task_struct (current); -} - -static void swap_read_folio_bdev_async (struct folio *folio, -struct swap_info_struct *sis) -{ -struct bio *bio; - - bio =bio_alloc (sis->bdev,1 ,REQ_OP_READ ,GFP_KERNEL ); - bio->bi_iter.bi_sector =swap_folio_sector (folio); - bio->bi_end_io = end_swap_bio_read; -bio_add_folio_nofail (bio, folio,folio_size (folio),0 ); -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); -count_memcg_folio_events (folio,PSWPIN ,folio_nr_pages (folio)); -count_vm_events (PSWPIN ,folio_nr_pages (folio)); -submit_bio (bio); -} -void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) {struct swap_info_struct *sis =__swap_entry_to_info (folio->swap);@@ -686,14 +490,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)/* We have to read from slower devices. Increase zswap protection. */ zswap_folio_swapin (folio); - -if (data_race (sis->flags &SWP_FS_OPS )) { -swap_read_folio_fs (ctx, folio); - }else if (synchronous) { -swap_read_folio_bdev_sync (folio, sis); - }else { -swap_read_folio_bdev_async (folio, sis); - } +swap_add_page (ctx, folio,READ ); finish:if (workingset) {@@ -703,18 +500,189 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)delayacct_swapin_end (); } -void swap_read_submit (struct swap_io_ctx *ctx) +static void swap_write_end (struct swap_iocb *sio,bool failed) +{ +int p; + +for (p =0 ; p < sio->nr_bvecs; p++) { +struct page *page = sio->bvecs[p].bv_page; + +if (failed) { +set_page_dirty (page); +ClearPageReclaim (page); + } +end_page_writeback (page); + } +mempool_free (sio, sio_pool); +} + +static void swap_fs_write_complete (struct kiocb *iocb,long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); +bool failed = ret != sio->len; + +if (failed) { +struct page *page = sio->bvecs[0 ].bv_page; + +/* +* In the case of swap-over-nfs, this can be a temporary failure +* if the system has limited memory for allocating transmit +* buffers. Mark the page dirty and avoid +* folio_rotate_reclaimable but rate-limit the messages. +*/ +pr_err_ratelimited ("Write error %ld on dio swapfile (%llu)\n" , + ret,swap_dev_pos (page_swap_entry (page))); + } + +swap_write_end (sio, failed); +} + +static void end_swap_bio_write (struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); +bool failed = !!bio->bi_status; + +if (failed) +pr_alert_ratelimited ("Write-error on swap-device (%u:%u:%llu)\n" , +MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), + (unsigned long long )bio->bi_iter.bi_sector); +bio_uninit (bio); +swap_write_end (sio, failed); +} + +static void swap_read_end (struct swap_iocb *sio,bool failed) +{ +int p; + +for (p =0 ; p < sio->nr_bvecs; p++) { +struct folio *folio =page_folio (sio->bvecs[p].bv_page); + +if (!failed) { +count_mthp_stat (folio_order (folio),MTHP_STAT_SWPIN ); +count_memcg_folio_events (folio,PSWPIN , +folio_nr_pages (folio)); +folio_mark_uptodate (folio); + } +folio_unlock (folio); + } + +if (!failed) +count_vm_events (PSWPIN , sio->len >>PAGE_SHIFT ); + +mempool_free (sio, sio_pool); +} + +static void swap_fs_read_complete (struct kiocb *iocb,long ret) +{ + struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); +bool failed = ret != sio->len; + +if (failed) +pr_alert_ratelimited ("Read-error on swap-device\n" ); +swap_read_end (sio, failed); +} + +static void swap_bio_read_end_io (struct bio *bio) +{ + struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio); +bool failed = !!bio->bi_status; + +if (failed) +pr_alert_ratelimited ("Read-error on swap-device (%u:%u:%llu)\n" , +MAJOR (bio_dev (bio)),MINOR (bio_dev (bio)), + (unsigned long long )bio->bi_iter.bi_sector); +bio_uninit (bio); +swap_read_end (sio, failed); +} + +static void swap_bdev_submit_write (struct swap_io_ctx *ctx) {struct swap_iocb *sio = ctx->sio; -struct iov_iter from; +struct bio *bio = &sio->bio; + +bio_init (bio, ctx->sis->bdev, sio->bvecs,ARRAY_SIZE (sio->bvecs), +REQ_OP_WRITE |REQ_SWAP ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector =swap_folio_sector (bio_first_folio_all (bio)); +bio_associate_blkg_from_page (bio,bio_first_folio_all (bio)); + +if (ctx->sis->flags &SWP_SYNCHRONOUS_IO ) { +submit_bio_wait (bio); +end_swap_bio_write (bio); + }else { + bio->bi_end_io = end_swap_bio_write; +submit_bio (bio); + } +} + +static void swap_bdev_submit_read (struct swap_io_ctx *ctx) +{ +struct swap_iocb *sio = ctx->sio; +struct bio *bio = &sio->bio; + +bio_init (bio, ctx->sis->bdev, sio->bvecs,ARRAY_SIZE (sio->bvecs), +REQ_OP_READ ); + bio->bi_iter.bi_size = sio->len; + bio->bi_iter.bi_sector =swap_folio_sector (bio_first_folio_all (bio)); + +if (ctx->sis->flags &SWP_SYNCHRONOUS_IO ) { +/* +* Keep this task valid during swap readpage because the oom +* killer may attempt to access it in the page fault retry +* time check. +*/ +get_task_struct (current); +submit_bio_wait (bio); +swap_bio_read_end_io (bio); +put_task_struct (current); + }else { + bio->bi_end_io = swap_bio_read_end_io; +submit_bio (bio); + } +} + +static void swap_fs_submit (struct swap_io_ctx *ctx,int rw) +{ +struct swap_iocb *sio = ctx->sio; +struct iov_iter iter;int ret; -if (!sio) -return ; +init_sync_kiocb (&sio->iocb, ctx->sis->swap_file); + sio->iocb.ki_pos =swap_dev_pos (page_folio (sio->bvecs[0 ].bv_page)->swap); +if (rw ==WRITE ) + sio->iocb.ki_complete = swap_fs_write_complete; +else + sio->iocb.ki_complete = swap_fs_read_complete; -iov_iter_bvec (&from,ITER_DEST , sio->bvecs, sio->nr_bvecs, sio->len); - ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &from); +iov_iter_bvec (&iter, rw ==WRITE ?ITER_SOURCE :ITER_DEST , + sio->bvecs, sio->nr_bvecs, sio->len); + ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &iter);if (ret != -EIOCBQUEUED ) -sio_read_complete (&sio->iocb, ret); + sio->iocb.ki_complete (&sio->iocb, ret); +} + +void swap_write_submit (struct swap_io_ctx *ctx) +{ +if (!ctx->sio) +return ; + +if (ctx->sis->flags &SWP_FS_OPS ) +swap_fs_submit (ctx,WRITE ); +else +swap_bdev_submit_write (ctx); + ctx->sio =NULL ; + ctx->sis =NULL ; +} + +void swap_read_submit (struct swap_io_ctx *ctx) +{ +if (!ctx->sio) +return ; + +if (ctx->sis->flags &SWP_FS_OPS ) +swap_fs_submit (ctx,READ ); +else +swap_bdev_submit_read (ctx); ctx->sio =NULL ; + ctx->sis =NULL ; }mm/swap.h +1
-0@@ -81,6 +81,7 @@ enum swap_cluster_flags {struct swap_io_ctx {struct swap_iocb *sio; +struct swap_info_struct *sis; };#ifdef CONFIG_SWAP mm/swapfile.c +4
-5@@ -2842,6 +2842,10 @@ static int setup_swap_extents(struct swap_info_struct *sis,struct inode *inode = mapping->host;int ret; + ret =sio_pool_init (); +if (ret) +return ret; +if (S_ISBLK (inode->i_mode)) { ret =add_swap_extent (sis,0 , sis->max,0 ); *span = sis->pages;@@ -2853,11 +2857,6 @@ static int setup_swap_extents(struct swap_info_struct *sis,if (ret <0 )return ret; sis->flags |=SWP_ACTIVATED ; -if ((sis->flags &SWP_FS_OPS ) && -sio_pool_init () !=0 ) { -destroy_swap_extents (sis, swap_file); -return -ENOMEM ; - }return ret; }Signature · 1 line
2.53.0
PATCH 5/8 mm/swap: remove count_swpout_vm_event
In reply to Christoph HellwigThere is only one caller left, so merge it into that.
--- mm/page_io.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-)
mm/page_io.c +10 -14
@@ -254,19 +254,6 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)return ret; } -static inline void count_swpout_vm_event (struct folio *folio) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -if (unlikely (folio_test_pmd_mappable (folio))) { -count_memcg_folio_events (folio,THP_SWPOUT ,1 ); -count_vm_event (THP_SWPOUT ); - } -#endif -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPOUT ); -count_memcg_folio_events (folio,PSWPOUT ,folio_nr_pages (folio)); -count_vm_events (PSWPOUT ,folio_nr_pages (folio)); -} -#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP )static struct cgroup_subsys_state *folio_memcg_blkg_css (struct folio *folio) {
@@ -388,7 +375,16 @@ void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) {VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -count_swpout_vm_event (folio); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +if (unlikely (folio_test_pmd_mappable (folio))) { +count_memcg_folio_events (folio,THP_SWPOUT ,1 ); +count_vm_event (THP_SWPOUT ); + } +#endif +count_mthp_stat (folio_order (folio),MTHP_STAT_SWPOUT ); +count_memcg_folio_events (folio,PSWPOUT ,folio_nr_pages (folio)); +count_vm_events (PSWPOUT ,folio_nr_pages (folio)); +folio_start_writeback (folio);folio_unlock (folio);swap_add_page (ctx, folio,WRITE );
Signature · 1 line
2.53.0
Re: [PATCH 5/8] mm/swap: remove count_swpout_vm_event
Quoted 6 lines · Christoph Hellwig
There is only one caller left, so merge it into that.
Signed-off-by Christoph Hellwig <[email protected]>--- mm/page_io.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-)
Quoted 46 lines
mm/page_io.c +10
-14@@ -254,19 +254,6 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)return ret; } -static inline void count_swpout_vm_event (struct folio *folio) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -if (unlikely (folio_test_pmd_mappable (folio))) { -count_memcg_folio_events (folio,THP_SWPOUT ,1 ); -count_vm_event (THP_SWPOUT ); - } -#endif -count_mthp_stat (folio_order (folio),MTHP_STAT_SWPOUT ); -count_memcg_folio_events (folio,PSWPOUT ,folio_nr_pages (folio)); -count_vm_events (PSWPOUT ,folio_nr_pages (folio)); -} -#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP )static struct cgroup_subsys_state *folio_memcg_blkg_css (struct folio *folio) {@@ -388,7 +375,16 @@ void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio) {VM_BUG_ON_FOLIO (!folio_test_swapcache (folio), folio); -count_swpout_vm_event (folio); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +if (unlikely (folio_test_pmd_mappable (folio))) { +count_memcg_folio_events (folio,THP_SWPOUT ,1 ); +count_vm_event (THP_SWPOUT ); + } +#endif +count_mthp_stat (folio_order (folio),MTHP_STAT_SWPOUT ); +count_memcg_folio_events (folio,PSWPOUT ,folio_nr_pages (folio)); +count_vm_events (PSWPOUT ,folio_nr_pages (folio)); +folio_start_writeback (folio);folio_unlock (folio);swap_add_page (ctx, folio,WRITE );Signature · 1 line
2.53.0
Re: [PATCH 5/8] mm/swap: remove count_swpout_vm_event
In reply to Christoph HellwigQuoted 4 lines · Christoph Hellwig
There is only one caller left, so merge it into that.
Signed-off-by Christoph Hellwig <[email protected]>
PATCH 6/8 mm/swap: use swap_ops to register swap device's methods
In reply to Christoph HellwigThis simplifies codes and makes logic clearer. And also makes later any new swap device type being added easier to handle.
Currently there are two types of swap devices: fs and bdev.
--- include/linux/swap.h | 1 + mm/page_io.c | 68 ++++++++++++++++++++++++++++---------------- mm/swap.h | 10 +++++++ mm/swapfile.c | 4 +++ 4 files changed, 58 insertions(+), 25 deletions(-)
include/linux/swap.h +1 -0
@@ -281,6 +281,7 @@ struct swap_info_struct {struct work_struct reclaim_work;/* reclaim worker */ struct list_head discard_clusters;/* discard clusters list */ struct plist_node avail_list;/* entry in swap_avail_head */ +const struct swap_ops *ops; };static inline swp_entry_t page_swap_entry (struct page *page)
mm/page_io.c +43 -25
96 lines not on this page · raw message
mm/swap.h +10 -0
@@ -84,6 +84,13 @@ struct swap_io_ctx {struct swap_info_struct *sis; }; +struct swap_ops { +bool (*can_merge)(struct folio *folio,struct folio *prev_folio, +size_t prev_folio_size,int rw); +void (*submit_write)(struct swap_io_ctx *ctx); +void (*submit_read)(struct swap_io_ctx *ctx); +}; +#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */
@@ -471,6 +478,9 @@ static inline unsigned int folio_swap_flags(struct folio *folio)#endif /* CONFIG_SWAP */ +extern const struct swap_ops swap_bdev_ops; +extern const struct swap_ops swap_fs_ops; +int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);
mm/swapfile.c +4 -0
@@ -2846,6 +2846,8 @@ static int setup_swap_extents(struct swap_info_struct *sis,if (ret)return ret; + sis->ops = &swap_bdev_ops; +if (S_ISBLK (inode->i_mode)) { ret =add_swap_extent (sis,0 , sis->max,0 ); *span = sis->pages;
@@ -2856,6 +2858,8 @@ static int setup_swap_extents(struct swap_info_struct *sis, ret = mapping->a_ops->swap_activate (sis, swap_file, span);if (ret <0 )return ret; +if (sis->flags &SWP_FS_OPS ) + sis->ops = &swap_fs_ops; sis->flags |=SWP_ACTIVATED ;return ret; }
Signature · 1 line
2.53.0
Re: [PATCH 6/8] mm/swap: use swap_ops to register swap device's methods
Quoted 10 lines · Christoph Hellwig
This simplifies codes and makes logic clearer. And also makes later any new swap device type being added easier to handle.
Currently there are two types of swap devices: fs and bdev.
Suggested-by Chris Li <[email protected]>Signed-off-by Baoquan He <[email protected]>[hch: updated for the new submit and can_merge abstraction]Signed-off-by Christoph Hellwig <[email protected]>
Baoquan He
reviewerHi Christoph,
Quoted 5 lines · Christoph Hellwig
Hi all,
this series makes use of the
swap_iocbfor block as well so that it doesn't do inefficient single-bio I/O, and then rebases theswap_opsfrom Baoquan on top of the now very different method structure.
What tree is this series based on? I tried the latest mainline, linux-next main branch and Andrew's mm-unstable, all failed.
And by the way, usually we add version number in each patch's subject as "[PATCH v2] xxx ".
Thanks
Baoquan
Quoted 6 lines · Baoquan He
Quoted 3 lines
this series makes use of the
swap_iocbfor block as well so that it doesn't do inefficient single-bio I/O, and then rebases theswap_opsfrom Baoquan on top of the now very different method structure.What tree is this series based on? I tried the latest mainline, linux-next main branch and Andrew's mm-unstable, all failed.
The base is:
commit e1af79f3291a268adf4e149e1faba3052743e898 (akpm/mm-unstable)
Author: Joshua Hahn <[email protected]> Date: Fri May 29 13:27:54 2026 -0700
mm/nodemask: correctly describe nodemask operation return types
Quoted 2 lines
And by the way, usually we add version number in each patch's subject as "[PATCH v2] xxx ".
That does sometimes happen but is rather unusual if you take a quick look over lkml. It also isn't the default for any normal patch sending tool.
Baoquan He
reviewerQuoted 8 lines · Christoph Hellwig
Quoted 6 lines · Baoquan He
Quoted 3 lines
this series makes use of the
swap_iocbfor block as well so that it doesn't do inefficient single-bio I/O, and then rebases theswap_opsfrom Baoquan on top of the now very different method structure.What tree is this series based on? I tried the latest mainline, linux-next main branch and Andrew's mm-unstable, all failed.
Quoted 7 lines
The base is:
commit
e1af79f3291a268adf4e149e1faba3052743e898(akpm/mm-unstable)Author: Joshua Hahn <[email protected]> Date: Fri May 29 13:27:54 2026 -0700mm/nodemask: correctly describe nodemask operation return types
Thanks. And seems the patchset pulled back via b4 only contains the patch 1~5/8 and patch 8/8, it misses 6~7/8. Just in case if someone else also use b4 and meets the similar issue when applying patches. This could be related to the email sending tool or its settings.
Quoted 8 lines
Quoted 4 lines
And by the way, usually we add version number in each patch's subject as "[PATCH v2] xxx ".
That does sometimes happen but is rather unusual if you take a quick look over lkml. It also isn't the default for any normal patch sending tool.
Really, what I have seen are all this kind of format, but surely I could see too few. Anyway, it doesn't matter much. Thanks.
Quoted 15 lines · Baoquan He
Quoted 14 lines · Christoph Hellwig
Quoted 6 lines · Baoquan He
Quoted 3 lines
this series makes use of the
swap_iocbfor block as well so that it doesn't do inefficient single-bio I/O, and then rebases theswap_opsfrom Baoquan on top of the now very different method structure.What tree is this series based on? I tried the latest mainline, linux-next main branch and Andrew's mm-unstable, all failed.
The base is:
commit
e1af79f3291a268adf4e149e1faba3052743e898(akpm/mm-unstable)Author: Joshua Hahn <[email protected]> Date: Fri May 29 13:27:54 2026 -0700
Quoted 6 lines
Quoted 1 line
mm/nodemask: correctly describe nodemask operation return types
Thanks. And seems the patchset pulled back via b4 only contains the patch 1~5/8 and patch 8/8, it misses 6~7/8. Just in case if someone else also use b4 and meets the similar issue when applying patches. This could be related to the email sending tool or its settings.
In case it makes your (or anyone elses) life easier, I also have a git branch here:
https://git.infradead.org/?p=users/hch/misc.git;a=shortlog;h=refs/heads/swap_ops
PATCH 7/8 mm/swap: remove SWP_FS_OPS
In reply to Christoph HellwigProvide a swap_fs_activate helper that directly sets up swap_fs_ops, and a flag in struct swap_ops to indicate of NOFS swapping is allowed.
--- Documentation/filesystems/locking.rst | 5 +++-- Documentation/filesystems/vfs.rst | 4 ++-- fs/nfs/file.c | 4 +--- fs/smb/client/file.c | 4 +--- include/linux/swap.h | 6 +++++- mm/page_io.c | 10 +++++++++- mm/swap.h | 16 ++++------------ mm/swapfile.c | 2 -- mm/vmscan.c | 14 ++++++-------- 9 files changed, 31 insertions(+), 34 deletions(-)
Documentation/filesystems/locking.rst +3 -2
@@ -355,13 +355,14 @@ should perform any validation and preparation necessary to ensure that writes can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted through -->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted +->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted directly to the block device ``sis->bdev``. ->swap_deactivate() will be called in the sys_swapoff() path after ->swap_activate() returned success. -->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate(). +->swap_rw will be called for swap IO if swap_fs_activate was called by +->swap_activate(). file_lock_operations ====================
Documentation/filesystems/vfs.rst +2 -2
@@ -977,7 +977,7 @@ cache in your filesystem. The following members are defined: can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted - through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will + through ->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted directly to the block device ``sis->bdev``. ``swap_deactivate``
@@ -985,7 +985,7 @@ cache in your filesystem. The following members are defined: successful. ``swap_rw`` - Called to read or write swap pages when SWP_FS_OPS is set. + Called to read or write swap pages when swap_fs_activate was called. The File Object ===============
fs/nfs/file.c +1 -3
@@ -589,7 +589,7 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, ret =rpc_clnt_swap_activate (clnt);if (ret)return ret; - ret =add_swap_extent (sis,0 , sis->max,0 ); + ret =swap_fs_activate (sis);if (ret <0 ) {rpc_clnt_swap_deactivate (clnt);return ret;
@@ -599,8 +599,6 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file,if (cl->rpc_ops->enable_swap) cl->rpc_ops->enable_swap (inode); - - sis->flags |=SWP_FS_OPS ;return ret; }
fs/smb/client/file.c +1 -3
@@ -3332,9 +3332,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis,* but we could add call to grab a byte range lock to prevent others * from reading or writing the file */ - - sis->flags |=SWP_FS_OPS ; -return add_swap_extent (sis,0 , sis->max,0 ); +return swap_fs_activate (sis); }static void cifs_swap_deactivate (struct file *file)
include/linux/swap.h +5 -1
@@ -208,7 +208,6 @@ enum {SWP_SOLIDSTATE = (1 <<4 ),/* blkdev seeks are cheap */ SWP_BLKDEV = (1 <<6 ),/* its a block device */ SWP_ACTIVATED = (1 <<7 ),/* set after swap_activate success */ -SWP_FS_OPS = (1 <<8 ),/* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 <<9 ),/* single-time swap area discards */ SWP_PAGE_DISCARD = (1 <<10 ),/* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
@@ -403,6 +402,7 @@ extern void __meminit kswapd_stop(int nid);#ifdef CONFIG_SWAP +int swap_fs_activate (struct swap_info_struct *sis);int add_swap_extent (struct swap_info_struct *sis,unsigned long start_page,unsigned long nr_pages,sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -527,6 +527,10 @@ static inline bool folio_free_swap(struct folio *folio)return false; } +static inline int swap_fs_activate (struct swap_info_struct *sis) +{ +return -EINVAL ; +}static inline int add_swap_extent (struct swap_info_struct *sis,unsigned long start_page,unsigned long nr_pages,sector_t start_block)
mm/page_io.c +9 -1
@@ -677,12 +677,20 @@ static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,swap_dev_pos (prev_folio->swap) + prev_folio_size; } -const struct swap_ops swap_fs_ops = { +static const struct swap_ops swap_fs_ops = { + .flags =SWAP_OPS_F_NOFS , .submit_write = swap_fs_submit_write, .submit_read = swap_fs_submit_read, .can_merge = swap_fs_can_merge, }; +int swap_fs_activate (struct swap_info_struct *sis) +{ + sis->ops = &swap_fs_ops; +return add_swap_extent (sis,0 , sis->max,0 ); +} +EXPORT_SYMBOL_GPL (swap_fs_activate); +void swap_write_submit (struct swap_io_ctx *ctx) {if (!ctx->sio)
mm/swap.h +4 -12
@@ -84,7 +84,11 @@ struct swap_io_ctx {struct swap_info_struct *sis; }; +#define SWAP_OPS_F_NOFS (1U << 0) +struct swap_ops { +unsigned int flags; +bool (*can_merge)(struct folio *folio,struct folio *prev_folio,size_t prev_folio_size,int rw);void (*submit_write)(struct swap_io_ctx *ctx);
@@ -335,11 +339,6 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders,void swap_update_readahead (struct folio *folio,struct vm_area_struct *vma,unsigned long addr); -static inline unsigned int folio_swap_flags (struct folio *folio) -{ -return __swap_entry_to_info (folio->swap)->flags; -} -#else /* CONFIG_SWAP */ static inline struct swap_cluster_info *swap_cluster_lock (struct swap_info_struct *si,pgoff_t offset,bool irq)
@@ -470,16 +469,9 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } - -static inline unsigned int folio_swap_flags (struct folio *folio) -{ -return 0 ; -} -#endif /* CONFIG_SWAP */ extern const struct swap_ops swap_bdev_ops; -extern const struct swap_ops swap_fs_ops;int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);
mm/swapfile.c +0 -2
@@ -2858,8 +2858,6 @@ static int setup_swap_extents(struct swap_info_struct *sis, ret = mapping->a_ops->swap_activate (sis, swap_file, span);if (ret <0 )return ret; -if (sis->flags &SWP_FS_OPS ) - sis->ops = &swap_fs_ops; sis->flags |=SWP_ACTIVATED ;return ret; }
mm/vmscan.c +6 -8
@@ -1037,16 +1037,14 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) {if (gfp_mask & __GFP_FS)return true; -if (!folio_test_swapcache (folio) || !(gfp_mask & __GFP_IO)) -return false;/* -* We can "enter_fs" for swap-cache with only __GFP_IO -* providing this isn't SWP_FS_OPS. -* ->flags can be updated non-atomically, -* but that will never affect SWP_FS_OPS, so the data_race -* is safe. +* We can "enter_fs" for swap-cache with only __GFP_IO unless backed by +* a swapfile that requires GFP_NOFS I/O. */ -return !data_race (folio_swap_flags (folio) &SWP_FS_OPS ); +if (folio_test_swapcache (folio) && (gfp_mask & __GFP_IO) && + !(__swap_entry_to_info (folio->swap)->ops->flags &SWAP_OPS_F_NOFS )) +return true; +return false; }/*
Signature · 1 line
2.53.0
Baoquan He
reviewerRe: [PATCH 7/8] mm/swap: remove SWP_FS_OPS
On 06/01/26 at 01:34pm, Christoph Hellwig wrote:
...snip...
mm/swap.h struct swap_io_ctx { · line 88
Quoted 10 lines
mm/swap.h +2
-0@@ -84,7 +84,11 @@ struct swap_io_ctx {struct swap_info_struct *sis; }; +#define SWAP_OPS_F_NOFS (1U << 0) +
The name of this flag and its usage are a little weird, surely the old one SWP_FS_OPS is similar. Do you think if we need add some document as below to make it clearer? Personal opinion and just for reference.
mm/swap.h +12 -0
@@ -84,6 +84,18 @@ struct swap_io_ctx {struct swap_info_struct *sis; }; +/* +* SWAP_OPS_F_NOFS: swap operations that may recurse into the filesystem +* (e.g. NFS, CIFS). When set, callers (specifically reclaim) must operate +* under GFP_NOFS / !__GFP_FS to avoid deadlock. When clear (block-device +* swap), the swap I/O path does not touch the filesystem, so reclaim may +* safely enter the filesystem even with only __GFP_IO. +* +* In may_enter_fs() this appears as a double negative (!SWAP_OPS_F_NOFS) +* which reads as "not a nofs swap => may enter fs". This is intentional: +* it is bdev swap (the common case) that allows filesystem re-entry, while +* filesystem-based swap explicitly forbids it. +*/ #define SWAP_OPS_F_NOFS (1U << 0)struct swap_ops {
mm/vmscan.c +3 -1
@@ -1039,7 +1039,9 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)return true;/* * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by -* a swapfile that requires GFP_NOFS I/O. +* a swapfile that requires GFP_NOFS I/O (i.e. filesystem-based swap +* like NFS/CIFS, which sets SWAP_OPS_F_NOFS). Block-device swap +* never touches the filesystem, so it is safe to enter fs here. */ if (folio_test_swapcache (folio) && (gfp_mask & __GFP_IO) && !(__swap_entry_to_info (folio->swap)->ops->flags &SWAP_OPS_F_NOFS ))
Quoted 78 lines
mm/swap.h +2
-12struct swap_ops { +unsigned int flags; +bool (*can_merge)(struct folio *folio,struct folio *prev_folio,size_t prev_folio_size,int rw);void (*submit_write)(struct swap_io_ctx *ctx);@@ -335,11 +339,6 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders,void swap_update_readahead (struct folio *folio,struct vm_area_struct *vma,unsigned long addr); -static inline unsigned int folio_swap_flags (struct folio *folio) -{ -return __swap_entry_to_info (folio->swap)->flags; -} -#else /* CONFIG_SWAP */ static inline struct swap_cluster_info *swap_cluster_lock (struct swap_info_struct *si,pgoff_t offset,bool irq)@@ -470,16 +469,9 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } - -static inline unsigned int folio_swap_flags (struct folio *folio) -{ -return 0 ; -} -#endif /* CONFIG_SWAP */ extern const struct swap_ops swap_bdev_ops; -extern const struct swap_ops swap_fs_ops;int shmem_writeout (struct swap_io_ctx *ctx,struct folio *folio,struct list_head *folio_list);mm/swapfile.c +0
-2@@ -2858,8 +2858,6 @@ static int setup_swap_extents(struct swap_info_struct *sis, ret = mapping->a_ops->swap_activate (sis, swap_file, span);if (ret <0 )return ret; -if (sis->flags &SWP_FS_OPS ) - sis->ops = &swap_fs_ops; sis->flags |=SWP_ACTIVATED ;return ret; }mm/vmscan.c +6
-8@@ -1037,16 +1037,14 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) {if (gfp_mask & __GFP_FS)return true; -if (!folio_test_swapcache (folio) || !(gfp_mask & __GFP_IO)) -return false;/* -* We can "enter_fs" for swap-cache with only __GFP_IO -* providing this isn't SWP_FS_OPS. -* ->flags can be updated non-atomically, -* but that will never affect SWP_FS_OPS, so the data_race -* is safe. +* We can "enter_fs" for swap-cache with only __GFP_IO unless backed by +* a swapfile that requires GFP_NOFS I/O. */ -return !data_race (folio_swap_flags (folio) &SWP_FS_OPS ); +if (folio_test_swapcache (folio) && (gfp_mask & __GFP_IO) && + !(__swap_entry_to_info (folio->swap)->ops->flags &SWAP_OPS_F_NOFS )) +return true; +return false; }/* Signature · 1 line
2.53.0
Re: [PATCH 7/8] mm/swap: remove SWP_FS_OPS
Quoted 11 lines · Baoquan He
+ *
SWAP_OPS_F_NOFS: swap operations that may recurse into the filesystem
+ * (e.g. NFS, CIFS). When set, callers (specifically reclaim) must operate
+ * underGFP_NOFS/ !__GFP_FS to avoid deadlock. When clear (block-device
+ * swap), the swap I/O path does not touch the filesystem, so reclaim may
+ * safely enter the filesystem even with only__GFP_IO.
+ *
+ * Inmay_enter_fs()this appears as a double negative (!SWAP_OPS_F_NOFS)
+ * which reads as "not a nofs swap => may enter fs". This is intentional:
+ * it is bdev swap (the common case) that allows filesystem re-entry, while
+ * filesystem-based swap explicitly forbids it.
+ */
This is very verbose, but not incorrect. If everyone is fine with it, I can add it to the next version.
Or if people think it is clearer. we can add a gfp constraint to the ops?
Re: [PATCH 7/8] mm/swap: remove SWP_FS_OPS
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
Provide a
swap_fs_activatehelper that directly sets upswap_fs_ops, and a flag in structswap_opsto indicate of NOFS swapping is allowed.Signed-off-by Christoph Hellwig <[email protected]>
Code LGTM, with Baoquan's documentation proposal. Tentatively:
PATCH 8/8 mm/vmstat: add NRSWP{IN,OUT} counters
In reply to Christoph HellwigCount how many swap I/Os we cause. Due to batching this can be different than the currently counter number of pages written/read, and tracking this information is useful to see how efficient the batching is.
--- include/linux/vm_event_item.h | 4 ++++ mm/page_io.c | 2 ++ mm/vmstat.c | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)
include/linux/vm_event_item.h +4 -0
@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST,#endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP +NRSWPIN , +NRSWPOUT , +#endif /* CONFIG_SWAP */NR_VM_EVENT_ITEMS };
mm/page_io.c +2 -0
@@ -695,6 +695,7 @@ void swap_write_submit(struct swap_io_ctx *ctx) {if (!ctx->sio)return ; +count_vm_events (NRSWPOUT ,1 ); ctx->sis->ops->submit_write (ctx); ctx->sio =NULL ; ctx->sis =NULL ;
@@ -704,6 +705,7 @@ void swap_read_submit(struct swap_io_ctx *ctx) {if (!ctx->sio)return ; +count_vm_events (NRSWPIN ,1 ); ctx->sis->ops->submit_read (ctx); ctx->sio =NULL ; ctx->sis =NULL ;
mm/vmstat.c +5 -1
@@ -1488,7 +1488,11 @@ const char * const vmstat_text[] = {#if THREAD_SIZE >65536 [I(KSTACK_REST)] = "kstack_rest",#endif -#endif +#endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + [I(NRSWPIN)] = "nrswpin", + [I(NRSWPOUT)] = "nrswpout", +#endif /* CONFIG_SWAP */ #undef I#endif /* CONFIG_VM_EVENT_COUNTERS */ };
Signature · 1 line
2.53.0
Baoquan He
reviewerRe: [PATCH 8/8] mm/vmstat: add NRSWP{IN,OUT} counters
Quoted 2 lines · Christoph Hellwig
Count how many swap I/Os we cause. Due to batching this can be different than the currently counter number of pages written/read,
~~~~~~~ current, typo?
Quoted 15 lines
and tracking this information is useful to see how efficient the batching is.
Signed-off-by Christoph Hellwig <[email protected]>--- include/linux/vm_event_item.h | 4 ++++ mm/page_io.c | 2 ++ mm/vmstat.c | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)include/linux/vm_event_item.h +0
-0
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · line 181
Quoted 7 lines
include/linux/vm_event_item.h +4
-0@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST,#endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */
Putting them together with 'SWAP_RA ... SWPOUT_ZERO' could impact the ongoing program/script?
Quoted 43 lines
include/linux/vm_event_item.h +0
-0NR_VM_EVENT_ITEMS };mm/page_io.c +2
-0@@ -695,6 +695,7 @@ void swap_write_submit(struct swap_io_ctx *ctx) {if (!ctx->sio)return ; +count_vm_events (NRSWPOUT ,1 ); ctx->sis->ops->submit_write (ctx); ctx->sio =NULL ; ctx->sis =NULL ;@@ -704,6 +705,7 @@ void swap_read_submit(struct swap_io_ctx *ctx) {if (!ctx->sio)return ; +count_vm_events (NRSWPIN ,1 ); ctx->sis->ops->submit_read (ctx); ctx->sio =NULL ; ctx->sis =NULL ;mm/vmstat.c +5
-1@@ -1488,7 +1488,11 @@ const char * const vmstat_text[] = {#if THREAD_SIZE >65536 [I(KSTACK_REST)] = "kstack_rest",#endif -#endif +#endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + [I(NRSWPIN)] = "nrswpin", + [I(NRSWPOUT)] = "nrswpout", +#endif /* CONFIG_SWAP */ #undef I#endif /* CONFIG_VM_EVENT_COUNTERS */ };Signature · 1 line
2.53.0
Re: [PATCH 8/8] mm/vmstat: add NRSWP{IN,OUT} counters
Quoted 8 lines · Baoquan He
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · line 181
Quoted 5 lines
#endif /*
CONFIG_DEBUG_STACK_USAGE*/include/linux/vm_event_item.h +4
-0+#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */ Putting them together with '
SWAP_RA...SWPOUT_ZERO' could impact the ongoing program/script?
I was under the impression that we should not reorder vmstate events?
If reordering the vmstat output is fine, I can group them.
Re: [PATCH 8/8] mm/vmstat: add NRSWP{IN,OUT} counters
In reply to Christoph HellwigQuoted 18 lines · Christoph Hellwig
Count how many swap I/Os we cause. Due to batching this can be different than the currently counter number of pages written/read, and tracking this information is useful to see how efficient the batching is.
Signed-off-by Christoph Hellwig <[email protected]>--- include/linux/vm_event_item.h | 4 ++++ mm/page_io.c | 2 ++ mm/vmstat.c | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-)include/linux/vm_event_item.h +0
-0
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · line 181
Quoted 7 lines
include/linux/vm_event_item.h +4
-0@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST,#endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */
Same question as Baoquan regarding merging this with the other CONFIG_SWAP section, but otherwise:
(Was also going to suggest documentation of these stats, but then I can't find any documentation for these vmstats at all. Oh well.)