swap_ops updates v2
Subsystems (14)
Hi all,
this series is a follow on to the swap ops series now in mm-unstable.
The first patch reintroduces direct folio writes for synchronous swap files, the other two remove the double indirect for file system based swap.
Changes since v1:
- include swap_ops.h in zswap.c
- fix a commit message typo
- add a commit log blurb to mention there are not functional changes in patch 2
PATCH 1/3 mm/swap: revert to single-folio writes for synchronous swap devices
Kairui Song reported that zram benefits from submitting each folio directly instead of batching up I/O because the classic LRU scanning benefits from clearing the folio writeback bit in the scan loop.
Accommodate that by kicking off writes for synchronous devices for each iteration.
--- mm/page_io.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
mm/page_io.c +10 -1
@@ -358,7 +358,16 @@ static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw) }bvec_set_folio (&sio->bvecs[sio->nr_bvecs], folio,folio_size (folio),0 ); sio->len +=folio_size (folio); -if (++sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs)) { + +/* +* Write out the iocb if we filled it, or if the device is synchronous. +* +* The latter is to work around expectations in the classic LRU code +* which make synchronous clearing of the folio writeback flag in the +* reclaim path beneficial. +*/ +if (++sio->nr_bvecs ==ARRAY_SIZE (sio->bvecs) || + (rw ==WRITE && (sis->flags &SWP_SYNCHRONOUS_IO ))) {if (rw ==WRITE )swap_write_submit (ctx);else
Signature · 1 line
2.53.0
PATCH 2/3 mm/swap: add a new swap_ops.h header to allow for pluggable swap ops
In reply to Christoph HellwigAdd a new header to declare the swap_iocb, swap_ops and swap_ctx to allow for swap_ops implementations outside of mm/page_io.c. This will be used to remove the double indirection for file system-based swap. There is no functional change, just a move of the declarations.
Note that there already is a swapops.h header, which is totally unrelated to struct swap_ops. The close naming is a bit unfortunate, but I could not think of a better name for this header.
--- MAINTAINERS | 1 + include/linux/swap_ops.h | 39 +++++++++++++++++++++++++++++++++++++++ mm/madvise.c | 1 + mm/page_io.c | 10 +--------- mm/shmem.c | 1 + mm/swap.h | 23 +---------------------- mm/swap_state.c | 1 + mm/vmscan.c | 1 + mm/zswap.c | 2 +- 9 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 include/linux/swap_ops.h
MAINTAINERS +1 -0
@@ -17240,6 +17240,7 @@ S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-swap F: Documentation/mm/swap-table.rst F: include/linux/swap.h +F: include/linux/swap_ops.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c
include/linux/swap_ops.h new file +39 -0
@@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MM_SWAP_OPS_H +#define _MM_SWAP_OPS_H + +#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */ + +struct swap_iocb { +union { +struct kiocb iocb; +struct bio bio; + }; +struct bio_vec bvecs[SWAP_CLUSTER_MAX ]; +int nr_bvecs; +int len; +}; + +struct swap_io_ctx { +struct swap_iocb *sio; +struct swap_info_struct *sis; +}; + +/* +* SWAP_OPS_F_REQUIRE_NOFS: +* When set, all reclaim operations must operated as GFS_NOFS and not +* just GFP_NOIO, as GFP_NOIO allocations could recourse into the +* file system backing this swap file. +*/ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { +unsigned int flags; + +bool (*can_merge)(struct folio *folio,struct folio *prev_folio, +size_t prev_folio_size,int rw); +void (*submit_write)(struct swap_io_ctx *ctx); +void (*submit_read)(struct swap_io_ctx *ctx); +}; + +#endif /* _MM_SWAP_OPS_H */
mm/madvise.c +1 -0
@@ -32,6 +32,7 @@#include <linux/leafops.h> #include <linux/shmem_fs.h> #include <linux/mmu_notifier.h> +#include <linux/swap_ops.h> #include <asm/tlb.h>
mm/page_io.c +1 -9
@@ -25,6 +25,7 @@#include <linux/sched/task.h> #include <linux/delayacct.h> #include <linux/zswap.h> +#include <linux/swap_ops.h> #include "swap.h" #include "swap_table.h"
@@ -300,15 +301,6 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)#define bio_associate_blkg_from_page (bio, folio) do { } while (0)#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ -struct swap_iocb { -union { -struct kiocb iocb; -struct bio bio; - }; -struct bio_vec bvecs[SWAP_CLUSTER_MAX ]; -int nr_bvecs; -int len; -};static mempool_t *sio_pool;int sio_pool_init (void )
mm/shmem.c +1 -0
@@ -41,6 +41,7 @@#include <linux/swapfile.h> #include <linux/iversion.h> #include <linux/unicode.h> +#include <linux/swap_ops.h> #include "swap.h" static struct vfsmount *shm_mnt__ro_after_init ;
mm/swap.h +1 -22
@@ -8,6 +8,7 @@struct mempolicy ;struct swap_iocb ;struct swap_memcg_table ; +struct swap_io_ctx ;#if defined(MAX_POSSIBLE_PHYSMEM_BITS )#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
@@ -77,28 +78,6 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; -struct swap_io_ctx { -struct swap_iocb *sio; -struct swap_info_struct *sis; -}; - -/* -* SWAP_OPS_F_REQUIRE_NOFS: -* When set, all reclaim operations must operated as GFS_NOFS and not -* just GFP_NOIO, as GFP_NOIO allocations could recourse into the -* file system backing this swap file. -*/ -#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) - -struct swap_ops { -unsigned int flags; - -bool (*can_merge)(struct folio *folio,struct folio *prev_folio, -size_t prev_folio_size,int rw); -void (*submit_write)(struct swap_io_ctx *ctx); -void (*submit_read)(struct swap_io_ctx *ctx); -}; -#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */
mm/swap_state.c +1 -0
@@ -23,6 +23,7 @@#include <linux/huge_mm.h> #include <linux/shmem_fs.h> #include <linux/sysctl.h> +#include <linux/swap_ops.h> #include "internal.h" #include "swap_table.h" #include "swap.h"
mm/vmscan.c +1 -0
@@ -58,6 +58,7 @@#include <linux/random.h> #include <linux/mmu_notifier.h> #include <linux/parser.h> +#include <linux/swap_ops.h> #include <asm/tlbflush.h> #include <asm/div64.h>
mm/zswap.c +1 -1
@@ -20,7 +20,7 @@#include <linux/spinlock.h> #include <linux/types.h> #include <linux/atomic.h> -#include <linux/swap.h> +#include <linux/swap_ops.h> #include <linux/crypto.h> #include <linux/scatterlist.h> #include <linux/mempolicy.h>
Signature · 1 line
2.53.0
Re: [PATCH 2/3] mm/swap: add a new swap_ops.h header to allow for pluggable swap ops
Quoted 11 lines · Christoph Hellwig
Add a new header to declare the
swap_iocb,swap_opsandswap_ctxto allow forswap_opsimplementations outside ofmm/page_io.c. This will be used to remove the double indirection for file system-based swap. There is no functional change, just a move of the declarations.Note that there already is a
swapops.hheader, which is totally unrelated to structswap_ops. The close naming is a bit unfortunate, but I could not think of a better name for this header.Signed-off-by Christoph Hellwig <[email protected]>
Chris
Quoted 203 lines
--- MAINTAINERS | 1 + include/linux/swap_ops.h | 39 +++++++++++++++++++++++++++++++++++++++ mm/madvise.c | 1 + mm/page_io.c | 10 +--------- mm/shmem.c | 1 + mm/swap.h | 23 +---------------------- mm/swap_state.c | 1 + mm/vmscan.c | 1 + mm/zswap.c | 2 +- 9 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 include/linux/swap_ops.hMAINTAINERS +1
-0@@ -17240,6 +17240,7 @@ S: Maintained F: Documentation/ABI/testing/sysfs-kernel-mm-swap F: Documentation/mm/swap-table.rst F: include/linux/swap.h +F: include/linux/swap_ops.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.cinclude/linux/swap_ops.h new file +39
-0@@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MM_SWAP_OPS_H +#define _MM_SWAP_OPS_H + +#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */ + +struct swap_iocb { +union { +struct kiocb iocb; +struct bio bio; + }; +struct bio_vec bvecs[SWAP_CLUSTER_MAX ]; +int nr_bvecs; +int len; +}; + +struct swap_io_ctx { +struct swap_iocb *sio; +struct swap_info_struct *sis; +}; + +/* +* SWAP_OPS_F_REQUIRE_NOFS: +* When set, all reclaim operations must operated as GFS_NOFS and not +* just GFP_NOIO, as GFP_NOIO allocations could recourse into the +* file system backing this swap file. +*/ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { +unsigned int flags; + +bool (*can_merge)(struct folio *folio,struct folio *prev_folio, +size_t prev_folio_size,int rw); +void (*submit_write)(struct swap_io_ctx *ctx); +void (*submit_read)(struct swap_io_ctx *ctx); +}; + +#endif /* _MM_SWAP_OPS_H */ mm/madvise.c +1
-0@@ -32,6 +32,7 @@#include <linux/leafops.h> #include <linux/shmem_fs.h> #include <linux/mmu_notifier.h> +#include <linux/swap_ops.h> #include <asm/tlb.h> mm/page_io.c +1
-9@@ -25,6 +25,7 @@#include <linux/sched/task.h> #include <linux/delayacct.h> #include <linux/zswap.h> +#include <linux/swap_ops.h> #include "swap.h" #include "swap_table.h" @@ -300,15 +301,6 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)#define bio_associate_blkg_from_page (bio, folio) do { } while (0)#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ -struct swap_iocb { -union { -struct kiocb iocb; -struct bio bio; - }; -struct bio_vec bvecs[SWAP_CLUSTER_MAX ]; -int nr_bvecs; -int len; -};static mempool_t *sio_pool;int sio_pool_init (void )mm/shmem.c +1
-0@@ -41,6 +41,7 @@#include <linux/swapfile.h> #include <linux/iversion.h> #include <linux/unicode.h> +#include <linux/swap_ops.h> #include "swap.h" static struct vfsmount *shm_mnt__ro_after_init ;mm/swap.h +1
-22@@ -8,6 +8,7 @@struct mempolicy ;struct swap_iocb ;struct swap_memcg_table ; +struct swap_io_ctx ;#if defined(MAX_POSSIBLE_PHYSMEM_BITS )#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)@@ -77,28 +78,6 @@ enum swap_cluster_flags { CLUSTER_FLAG_MAX, }; -struct swap_io_ctx { -struct swap_iocb *sio; -struct swap_info_struct *sis; -}; - -/* -* SWAP_OPS_F_REQUIRE_NOFS: -* When set, all reclaim operations must operated as GFS_NOFS and not -* just GFP_NOIO, as GFP_NOIO allocations could recourse into the -* file system backing this swap file. -*/ -#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) - -struct swap_ops { -unsigned int flags; - -bool (*can_merge)(struct folio *folio,struct folio *prev_folio, -size_t prev_folio_size,int rw); -void (*submit_write)(struct swap_io_ctx *ctx); -void (*submit_read)(struct swap_io_ctx *ctx); -}; -#ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ mm/swap_state.c +1
-0@@ -23,6 +23,7 @@#include <linux/huge_mm.h> #include <linux/shmem_fs.h> #include <linux/sysctl.h> +#include <linux/swap_ops.h> #include "internal.h" #include "swap_table.h" #include "swap.h" mm/vmscan.c +1
-0@@ -58,6 +58,7 @@#include <linux/random.h> #include <linux/mmu_notifier.h> #include <linux/parser.h> +#include <linux/swap_ops.h> #include <asm/tlbflush.h> #include <asm/div64.h> mm/zswap.c +1
-2@@ -20,7 +20,7 @@#include <linux/spinlock.h> #include <linux/types.h> #include <linux/atomic.h> -#include <linux/swap.h> +#include <linux/swap_ops.h> #include <linux/crypto.h> #include <linux/scatterlist.h> #include <linux/mempolicy.h> --2.53.0
PATCH 3/3 mm/swap: move swap_ops into file systems for file system-based swap
In reply to Christoph HellwigCurrently swap to and from file systems goes through two indirect calls between the swap ops and the swap_rw method. Reduce this by directly providing the swap_ops from the file system.
For this refactor swap_fs_submit into a swap_fs_prepare_rw helper that initializes the iov_iter on the callers stack so that file systems can call it directly, and use that to initialize file system specific ops in the NFS and SMB clients, which then get passed to swap_fs_activate.
--- Documentation/filesystems/locking.rst | 9 +--- Documentation/filesystems/vfs.rst | 8 +--- fs/nfs/direct.c | 20 --------- fs/nfs/file.c | 42 ++++++++++++++++-- fs/smb/client/file.c | 63 +++++++++++++++++---------- include/linux/fs.h | 1 - include/linux/nfs_fs.h | 1 - include/linux/swap.h | 6 --- include/linux/swap_ops.h | 5 +++ mm/page_io.c | 34 +++------------ 10 files changed, 93 insertions(+), 96 deletions(-)
Documentation/filesystems/locking.rst +2 -7
@@ -266,7 +266,6 @@ prototypes:: int (*error_remove_folio)(struct address_space *, struct folio *); int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span) int (*swap_deactivate)(struct file *); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); locking rules: All except dirty_folio and free_folio may block
@@ -291,7 +290,6 @@ is_partially_uptodate: yes error_remove_folio: yes swap_activate: no swap_deactivate: no -swap_rw: yes, unlocks ====================== ======================== ========= =============== ->write_begin(), ->write_end() and ->read_folio() may be called from
@@ -355,15 +353,12 @@ should perform any validation and preparation necessary to ensure that writes can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted through -->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted -directly to the block device ``sis->bdev``. +the file system it should call swap_fs_activate, otherwise IO will be +submitted directly to the block device ``sis->bdev``. ->swap_deactivate() will be called in the sys_swapoff() path after ->swap_activate() returned success. -->swap_rw will be called for swap IO if swap_fs_activate was called by -->swap_activate(). - file_lock_operations ====================
Documentation/filesystems/vfs.rst +2 -6
@@ -776,7 +776,6 @@ cache in your filesystem. The following members are defined: int (*error_remove_folio)(struct mapping *mapping, struct folio *); int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span) int (*swap_deactivate)(struct file *); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); }; ``read_folio``
@@ -977,16 +976,13 @@ cache in your filesystem. The following members are defined: can be performed with minimal memory allocation. It should call add_swap_extent(), or the helper iomap_swapfile_activate(), and return the number of extents added. If IO should be submitted - through ->swap_rw(), it should call swap_fs_activate, otherwise IO will - be submitted directly to the block device ``sis->bdev``. + through the file system it should call swap_fs_activate, otherwise IO + will be submitted directly to the block device ``sis->bdev``. ``swap_deactivate`` Called during swapoff on files where swap_activate was successful. -``swap_rw`` - Called to read or write swap pages when swap_fs_activate was called. - The File Object ===============
fs/nfs/direct.c +0 -20
@@ -145,26 +145,6 @@ static void nfs_direct_file_adjust_size_locked(struct inode *inode, } } -/** -* nfs_swap_rw - NFS address space operation for swap I/O -* @iocb: target I/O control block -* @iter: I/O buffer -* -* Perform IO to the swap-file. This is much like direct IO. -*/ -int nfs_swap_rw (struct kiocb *iocb,struct iov_iter *iter) -{ -ssize_t ret; - -if (iov_iter_rw (iter) ==READ ) - ret =nfs_file_direct_read (iocb, iter, true); -else - ret =nfs_file_direct_write (iocb, iter, true); -if (ret <0 ) -return ret; -return 0 ; -} -static void nfs_direct_release_pages (struct page **pages,unsigned int npages) {unsigned int i;
fs/nfs/file.c +38 -4
73 lines not on this page · raw message
fs/smb/client/file.c +39 -24
99 lines not on this page · raw message
include/linux/fs.h +0 -1
@@ -438,7 +438,6 @@ struct address_space_operations {int (*swap_activate)(struct swap_info_struct *sis,struct file *file,sector_t *span);void (*swap_deactivate)(struct file *file); -int (*swap_rw)(struct kiocb *iocb,struct iov_iter *iter); };extern const struct address_space_operations empty_aops;
include/linux/nfs_fs.h +0 -1
@@ -548,7 +548,6 @@ static inline const struct cred *nfs_file_cred(struct file *file)/* * linux/fs/nfs/direct.c */ -int nfs_swap_rw (struct kiocb *iocb,struct iov_iter *iter);ssize_t nfs_file_direct_read (struct kiocb *iocb,struct iov_iter *iter,bool swap); ssize_t nfs_file_direct_write(struct kiocb *iocb,
include/linux/swap.h +0 -6
@@ -334,8 +334,6 @@ extern void __meminit kswapd_run(int nid);extern void __meminit kswapd_stop (int nid);#ifdef CONFIG_SWAP - -int swap_fs_activate (struct swap_info_struct *sis);int add_swap_extent (struct swap_info_struct *sis,unsigned long start_page,unsigned long nr_pages,sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -461,10 +459,6 @@ static inline bool folio_free_swap(struct folio *folio)return false; } -static inline int swap_fs_activate (struct swap_info_struct *sis) -{ -return -EINVAL ; -}static inline int add_swap_extent (struct swap_info_struct *sis,unsigned long start_page,unsigned long nr_pages,sector_t start_block)
include/linux/swap_ops.h +5 -0
@@ -36,4 +36,9 @@ struct swap_ops { void (*submit_read)(struct swap_io_ctx *ctx); }; +void swap_fs_prepare_rw (struct swap_io_ctx *ctx,int rw,struct iov_iter *iter); +bool swap_fs_can_merge (struct folio *folio,struct folio *prev_folio, +size_t prev_folio_size,int rw); +int swap_fs_activate (struct swap_info_struct *sis,const struct swap_ops *ops); +#endif /* _MM_SWAP_OPS_H */
mm/page_io.c +7 -27
@@ -650,11 +650,9 @@ const struct swap_ops swap_bdev_ops = { .can_merge = swap_bdev_can_merge, }; -static void swap_fs_submit (struct swap_io_ctx *ctx,int rw) +void swap_fs_prepare_rw (struct swap_io_ctx *ctx,int rw,struct iov_iter *iter) {struct swap_iocb *sio = ctx->sio; -struct iov_iter iter; -int ret;init_sync_kiocb (&sio->iocb, ctx->sis->swap_file); sio->iocb.ki_pos =swap_dev_pos (bvec_folio (&sio->bvecs[0 ])->swap);
@@ -663,40 +661,22 @@ static void swap_fs_submit(struct swap_io_ctx *ctx, int rw)else sio->iocb.ki_complete = swap_fs_read_complete; -iov_iter_bvec (&iter, rw ==WRITE ?ITER_SOURCE :ITER_DEST , +iov_iter_bvec (iter, rw ==WRITE ?ITER_SOURCE :ITER_DEST , sio->bvecs, sio->nr_bvecs, sio->len); - ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw (&sio->iocb, &iter); -if (ret != -EIOCBQUEUED ) - sio->iocb.ki_complete (&sio->iocb, ret); } +EXPORT_SYMBOL_GPL (swap_fs_prepare_rw); -static void swap_fs_submit_write (struct swap_io_ctx *ctx) -{ -swap_fs_submit (ctx,WRITE ); -} - -static void swap_fs_submit_read (struct swap_io_ctx *ctx) -{ -swap_fs_submit (ctx,READ ); -} - -static bool swap_fs_can_merge (struct folio *folio,struct folio *prev_folio, +bool swap_fs_can_merge (struct folio *folio,struct folio *prev_folio,size_t prev_folio_size,int rw) {return swap_dev_pos (folio->swap) ==swap_dev_pos (prev_folio->swap) + prev_folio_size; } +EXPORT_SYMBOL_GPL (swap_fs_can_merge); -static const struct swap_ops swap_fs_ops = { - .flags =SWAP_OPS_F_REQUIRE_NOFS , - .submit_write = swap_fs_submit_write, - .submit_read = swap_fs_submit_read, - .can_merge = swap_fs_can_merge, -}; - -int swap_fs_activate (struct swap_info_struct *sis) +int swap_fs_activate (struct swap_info_struct *sis,const struct swap_ops *ops) { - sis->ops = &swap_fs_ops; + sis->ops = ops;return add_swap_extent (sis,0 , sis->max,0 ); }EXPORT_SYMBOL_GPL (swap_fs_activate);
Signature · 1 line
2.53.0
Kairui Song
Acked-byQuoted 9 lines · Christoph Hellwig
Hi all,
this series is a follow on to the swap ops series now in mm-unstable.
The first patch reintroduces direct folio writes for synchronous swap files, the other two remove the double indirect for file system based swap.
Quoted 5 lines
Changes since v1:
- includeswap_ops.hinzswap.c
- fix a commit message typo
- add a commit log blurb to mention there are not functional changes in patch 2
This series looks good to me and we should have them, especially patch 1 to ensure existing workloads won't hit a regression, minor conflict in patch 2 for unstable or new, due to
https://lore.kernel.org/all/[email protected]/,
but should be easy to fix. And for the series: