mm/swap: introduce folio_swap_entry() and convert all folio->swap readers
Subsystems (22)
A message that is not in the archive (sent off-list or not yet received): [email protected]
RFC PATCH 1/6 mm/swap: introduce folio_swap_entry() and convert all folio->swap readers
Hugetlb folios keep their per-folio flags in folio->private, which shares a union with folio->swap. While such a folio sits in the swap cache, the low bits of folio->swap.val can therefore carry flag bits rather than zeroes - with HVO enabled, HPG_vmemmap_optimized (bit 4) is set on virtually every hugetlb folio. Any naked read of folio->swap on a swap-cached hugetlb folio would hand a corrupted swap entry to the swap table, zswap, the memcg accounting or the I/O paths.
Swap entries are always allocated aligned to the folio size, so rounding the raw value down to folio_nr_pages() recovers the real entry. Wrap that in folio_swap_entry() and convert every reader of folio->swap in the tree to use it. For all non-hugetlb folios the low bits are always zero (order-0 folios trivially, large folios because the alignment leaves them clear), so the helper is an identity transform and this patch changes no behaviour on any existing path; reviewing it only requires auditing the helper itself plus the single invariant "swap entries are folio-size aligned".
The write side keeps raw access deliberately:__swap_cache_do_add_folio() now preserves the low flag bits when installing the entry, and __swap_cache_do_del_folio() clears the entry while keeping the flags, rounding the passed entry down so an unaligned value can never leak into the swap table.
This is preparation for swap support of hugetlb folios; hugetlb folios too small for all HPAGEFLAG bits to fit below the swap offset will be kept out of the swap cache by the feature's size gate.
--- include/linux/swap.h | 22 +++++++++++++++++++++- mm/huge_memory.c | 2 +- mm/memcontrol-v1.c | 4 ++-- mm/memcontrol.c | 2 +- mm/memory.c | 2 +- mm/page_io.c | 18 +++++++++--------- mm/shmem.c | 6 +++--- mm/swap.h | 8 +++++--- mm/swap_state.c | 15 ++++++++++----- mm/swapfile.c | 14 ++++++++------ mm/userfaultfd.c | 2 +- mm/util.c | 2 +- mm/vmscan.c | 4 ++-- mm/zswap.c | 4 ++-- 14 files changed, 67 insertions(+), 38 deletions(-)
include/linux/swap.h +21 -1
@@ -278,10 +278,30 @@ struct swap_info_struct {const struct swap_ops *ops; }; +/** +* folio_swap_entry - Get the swap entry recorded in a swap cache folio. +* @folio: The folio, must be in the swap cache. +* +* Hugetlb folios keep their per-folio flags in folio->private, which aliases +* folio->swap, so the low bits of folio->swap.val may carry flag bits while +* the folio is swap cached. Swap entries are always aligned to the folio +* size, so rounding down recovers the entry. Hugetlb folios too small for +* all flag bits to fit below the swap offset never enter the swap cache; +* see hugetlb_folio_swap_supported(). +* +* For all other folios the low bits are always zero, so this is an identity +* transform; always use it instead of reading folio->swap directly. +*/ +static inline swp_entry_t folio_swap_entry (const struct folio *folio) +{ +return (swp_entry_t ) { .val =round_down (folio->swap.val, +folio_nr_pages (folio)) }; +} +static inline swp_entry_t page_swap_entry (struct page *page) {struct folio *folio =page_folio (page); -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio); entry.val +=folio_page_idx (folio, page);return entry;
mm/huge_memory.c +1 -1
@@ -3760,7 +3760,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order,VM_WARN_ON_ONCE_PAGE (new_folio->private, new_head);if (folio_test_swapcache (folio)) - new_folio->swap.val = folio->swap.val + i; + new_folio->swap.val =folio_swap_entry (folio).val + i;/* Page flags must be visible before we make the page non-compound. */ smp_wmb ();
mm/memcontrol-v1.c +2 -2
@@ -299,7 +299,7 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci) swap_memcg =mem_cgroup_private_id_get_online (memcg, nr_entries);mod_memcg_state (swap_memcg,MEMCG_SWAP , nr_entries); -__swap_cgroup_set (ci,swp_cluster_offset (folio->swap), nr_entries, +__swap_cgroup_set (ci,swp_cluster_offset (folio_swap_entry (folio)), nr_entries,mem_cgroup_private_id (swap_memcg));folio_unqueue_deferred_split (folio);
@@ -369,7 +369,7 @@ void memcg1_swapin(struct folio *folio)*/ nr_pages =folio_nr_pages (folio); ci =swap_cluster_get_and_lock (folio); - id =__swap_cgroup_clear (ci,swp_cluster_offset (folio->swap), + id =__swap_cgroup_clear (ci,swp_cluster_offset (folio_swap_entry (folio)), nr_pages);swap_cluster_unlock (ci);mem_cgroup_uncharge_swap (id, nr_pages);
mm/memcontrol.c +1 -1
@@ -5781,7 +5781,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio)mod_memcg_state (memcg,MEMCG_SWAP , nr_pages); ci =swap_cluster_get_and_lock (folio); -__swap_cgroup_set (ci,swp_cluster_offset (folio->swap), nr_pages, +__swap_cgroup_set (ci,swp_cluster_offset (folio_swap_entry (folio)), nr_pages,mem_cgroup_private_id (memcg));swap_cluster_unlock (ci);
mm/memory.c +1 -1
@@ -5056,7 +5056,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) address = folio_start; ptep = folio_ptep; nr_pages = nr; - entry = folio->swap; + entry =folio_swap_entry (folio); page = &folio->page; }
mm/page_io.c +9 -9
@@ -160,7 +160,7 @@ static void swap_zeromap_folio_set(struct folio *folio)struct obj_cgroup *objcg =get_obj_cgroup_from_folio (folio);int nr_pages =folio_nr_pages (folio);struct swap_cluster_info *ci; -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio);unsigned int i;VM_WARN_ON_ONCE_FOLIO (!folio_test_swapcache (folio), folio);
@@ -183,7 +183,7 @@ static void swap_zeromap_folio_set(struct folio *folio)static void swap_zeromap_folio_clear (struct folio *folio) {struct swap_cluster_info *ci; -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio);unsigned int i;VM_WARN_ON_ONCE_FOLIO (!folio_test_swapcache (folio), folio);
@@ -321,7 +321,7 @@ int sio_pool_init(void)static bool swap_can_merge (struct swap_io_ctx *ctx,struct folio *folio,int rw) { -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct swap_info_struct *sis =__swap_entry_to_info (folio_swap_entry (folio));struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs -1 ];struct folio *prev_folio =bvec_folio (last_bv);size_t prev_folio_size =folio_size (prev_folio);
@@ -333,7 +333,7 @@ static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio,static void swap_add_folio (struct swap_io_ctx *ctx,struct folio *folio,int rw) { -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct swap_info_struct *sis =__swap_entry_to_info (folio_swap_entry (folio));struct swap_iocb *sio = ctx->sio;if (sio && !swap_can_merge (ctx, folio, rw)) {
@@ -432,7 +432,7 @@ static bool swap_read_folio_zeromap(struct folio *folio)* that an IO error is emitted (e.g. do_swap_page() will sigbus). * Folio lock stabilizes the cluster and map, so the check is safe. */ -if (WARN_ON_ONCE (swap_zeromap_batch (folio->swap, nr_pages, +if (WARN_ON_ONCE (swap_zeromap_batch (folio_swap_entry (folio), nr_pages, &is_zeromap) != nr_pages))return true;
@@ -453,7 +453,7 @@ static bool swap_read_folio_zeromap(struct folio *folio)void swap_read_folio (struct swap_io_ctx *ctx,struct folio *folio) { -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct swap_info_struct *sis =__swap_entry_to_info (folio_swap_entry (folio));bool synchronous = sis->flags &SWP_SYNCHRONOUS_IO ;bool workingset =folio_test_workingset (folio);unsigned long pflags;
@@ -525,7 +525,7 @@ static void swap_fs_write_complete(struct kiocb *iocb, long ret)* folio_rotate_reclaimable but rate-limit the messages. */ pr_err_ratelimited ("Write error %ld on dio swapfile (%llu)\n" , - ret,swap_dev_pos (folio->swap)); + ret,swap_dev_pos (folio_swap_entry (folio))); }swap_write_end (sio, failed);
@@ -671,8 +671,8 @@ EXPORT_SYMBOL_GPL(swap_fs_prepare_rw);bool swap_fs_can_merge (struct folio *folio,struct folio *prev_folio,size_t prev_folio_size,int rw) { -return swap_dev_pos (folio->swap) == -swap_dev_pos (prev_folio->swap) + prev_folio_size; +return swap_dev_pos (folio_swap_entry (folio)) == +swap_dev_pos (folio_swap_entry (prev_folio)) + prev_folio_size; }EXPORT_SYMBOL_GPL (swap_fs_can_merge);
mm/shmem.c +3 -3
@@ -1914,7 +1914,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, }folio_dup_swap (folio,NULL ); -shmem_delete_from_page_cache (folio,swp_to_radix_entry (folio->swap)); +shmem_delete_from_page_cache (folio,swp_to_radix_entry (folio_swap_entry (folio)));BUG_ON (folio_mapped (folio)); error =swap_writeout (ctx, folio);
@@ -1929,7 +1929,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,* it will be appropriate if other reactivate cases are added. */ error =shmem_add_to_page_cache (folio, mapping, index, -swp_to_radix_entry (folio->swap), +swp_to_radix_entry (folio_swap_entry (folio)), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);/* Swap entry might be erased by racing shmem_free_swap() */ if (!error) {
@@ -2293,7 +2293,7 @@ static int shmem_replace_folio(struct folio **foliop, gfp_t gfp, {struct swap_cluster_info *ci;struct folio *new, *old = *foliop; -swp_entry_t entry = old->swap; +swp_entry_t entry =folio_swap_entry (old);int nr_pages =folio_nr_pages (old);int error =0 ;
mm/swap.h +5 -3
@@ -173,10 +173,12 @@ static inline struct swap_cluster_info *swap_cluster_lock(static inline struct swap_cluster_info *__swap_cluster_get_and_lock (const struct folio *folio,bool irq) { +swp_entry_t entry =folio_swap_entry (folio); +VM_WARN_ON_ONCE_FOLIO (!folio_test_locked (folio), folio);VM_WARN_ON_ONCE_FOLIO (!folio_test_swapcache (folio), folio); -return __swap_cluster_lock (__swap_entry_to_info (folio->swap), -swp_offset (folio->swap), irq); +return __swap_cluster_lock (__swap_entry_to_info (entry), +swp_offset (entry), irq); }/*
@@ -287,7 +289,7 @@ static inline loff_t swap_dev_pos(swp_entry_t entry)static inline bool folio_matches_swap_entry (const struct folio *folio,swp_entry_t entry) { -swp_entry_t folio_entry = folio->swap; +swp_entry_t folio_entry =folio_swap_entry (folio);long nr_pages =folio_nr_pages (folio);VM_WARN_ON_ONCE_FOLIO (!folio_test_locked (folio), folio);
mm/swap_state.c +10 -5
@@ -227,7 +227,11 @@ static void __swap_cache_do_add_folio(struct swap_cluster_info *ci,folio_ref_add (folio, nr_pages);folio_set_swapcache (folio); - folio->swap = entry; +/* +* Keep the low bits, they may carry hugetlb specific flags that +* share folio->private with folio->swap (see folio_swap_entry()). +*/ + folio->swap.val = entry.val | (folio->swap.val & (nr_pages -1 )); }/**
@@ -269,6 +273,7 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,VM_WARN_ON_ONCE_FOLIO (!folio_test_swapcache (folio), folio);VM_WARN_ON_ONCE_FOLIO (folio_test_writeback (folio), folio); + entry.val =round_down (entry.val, nr_pages); si =__swap_entry_to_info (entry); ci_start =swp_cluster_offset (entry); ci_end = ci_start + nr_pages;
@@ -286,7 +291,7 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci, __swp_tb_get_flags(old_tb))); }while (++ci_off < ci_end); - folio->swap.val =0 ; + folio->swap.val &= nr_pages -1 ;folio_clear_swapcache (folio);if (!folio_swapped) {
@@ -336,7 +341,7 @@ void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio,void swap_cache_del_folio (struct folio *folio) {struct swap_cluster_info *ci; -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio); ci =swap_cluster_lock (__swap_entry_to_info (entry),swp_offset (entry));__swap_cache_del_folio (ci, folio, entry,NULL );
@@ -362,7 +367,7 @@ void swap_cache_del_folio(struct folio *folio)void __swap_cache_replace_folio (struct swap_cluster_info *ci,struct folio *old,struct folio *new) { -swp_entry_t entry = new->swap; +swp_entry_t entry =folio_swap_entry (new);unsigned long nr_pages =folio_nr_pages (new);unsigned int ci_off =swp_cluster_offset (entry);unsigned int ci_end = ci_off + nr_pages;
@@ -387,7 +392,7 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,*/ if (IS_ENABLED (CONFIG_DEBUG_VM ) &&folio_order (old) !=folio_order (new)) { - ci_off =swp_cluster_offset (old->swap); + ci_off =swp_cluster_offset (folio_swap_entry (old)); ci_end = ci_off +folio_nr_pages (old);while (ci_off++ < ci_end)WARN_ON_ONCE (swp_tb_to_folio (__swap_table_get (ci, ci_off)) != old);
mm/swapfile.c +8 -6
@@ -230,7 +230,7 @@ static int __try_to_reclaim_swap(struct swap_info_struct *si,folio_put (folio); goto again; } - offset =swp_offset (folio->swap); + offset =swp_offset (folio_swap_entry (folio)); need_reclaim = ((flags & TTRS_ANYWAY) || ((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) ||
@@ -330,12 +330,14 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)sector_t swap_folio_sector (struct folio *folio) { -struct swap_info_struct *sis =__swap_entry_to_info (folio->swap); +struct swap_info_struct *sis;struct swap_extent *se;sector_t sector; +swp_entry_t entry =folio_swap_entry (folio);pgoff_t offset; - offset =swp_offset (folio->swap); + sis =__swap_entry_to_info (entry); + offset =swp_offset (entry); se =offset_to_swap_extent (sis, offset); sector = se->start_block + (offset - se->start_page);return sector << (PAGE_SHIFT -9 );
@@ -1803,7 +1805,7 @@ int folio_alloc_swap(struct folio *folio)*/ int folio_dup_swap (struct folio *folio,struct page *page) { -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio);unsigned long nr_pages =folio_nr_pages (folio);VM_WARN_ON_FOLIO (!folio_test_locked (folio), folio);
@@ -1829,7 +1831,7 @@ int folio_dup_swap(struct folio *folio, struct page *page)*/ void folio_put_swap (struct folio *folio,struct page *page) { -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio);unsigned long nr_pages =folio_nr_pages (folio);struct swap_info_struct *si =__swap_entry_to_info (entry);
@@ -2033,7 +2035,7 @@ int swp_swapcount(swp_entry_t entry)*/ static bool folio_maybe_swapped (struct folio *folio) { -swp_entry_t entry = folio->swap; +swp_entry_t entry =folio_swap_entry (folio);struct swap_cluster_info *ci;unsigned int ci_off, ci_end;bool ret = false;
mm/userfaultfd.c +1 -1
@@ -1383,7 +1383,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,* not locked. */ if (src_folio &&unlikely (!folio_test_swapcache (src_folio) || - entry.val != src_folio->swap.val)) + entry.val !=folio_swap_entry (src_folio).val))return -EAGAIN ;double_pt_lock (dst_ptl, src_ptl);
mm/util.c +1 -1
@@ -727,7 +727,7 @@ struct address_space *folio_mapping(const struct folio *folio)return NULL ;if (unlikely (folio_test_swapcache (folio))) -return swap_address_space (folio->swap); +return swap_address_space (folio_swap_entry (folio)); mapping = folio->mapping;if ((unsigned long )mapping &FOLIO_MAPPING_FLAGS )
mm/vmscan.c +2 -2
@@ -731,7 +731,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, }if (folio_test_swapcache (folio)) { -swp_entry_t swap = folio->swap; +swp_entry_t swap =folio_swap_entry (folio);if (reclaimed && !mapping_exiting (mapping)) shadow =workingset_eviction (folio, target_memcg);
@@ -1049,7 +1049,7 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)* a swapfile that requires GFP_NOFS I/O. */ if (folio_test_swapcache (folio) && (gfp_mask & __GFP_IO) && - !(__swap_entry_to_info (folio->swap)->ops->flags & + !(__swap_entry_to_info (folio_swap_entry (folio))->ops->flags &SWAP_OPS_F_REQUIRE_NOFS ))return true;return false;
mm/zswap.c +2 -2
@@ -1475,7 +1475,7 @@ static bool zswap_store_page(struct page *page,bool zswap_store (struct folio *folio) {long nr_pages =folio_nr_pages (folio); -swp_entry_t swp = folio->swap; +swp_entry_t swp =folio_swap_entry (folio);struct obj_cgroup *objcg =NULL ;struct mem_cgroup *memcg =NULL ;struct zswap_pool *pool;
@@ -1580,7 +1580,7 @@ bool zswap_store(struct folio *folio)*/ int zswap_load (struct folio *folio) { -swp_entry_t swp = folio->swap; +swp_entry_t swp =folio_swap_entry (folio);pgoff_t offset =swp_offset (swp);struct xarray *tree =swap_zswap_tree (swp);struct zswap_entry *entry;
Signature · 1 line
2.53.0
RFC PATCH 2/6 mm/hugetlb: swap-in support for anonymous hugetlb folios
In reply to a message not in the archiveTeach the hugetlb fault path to recognize swap entries and read them back, together with the full consumer-side lifecycle of such entries. This is the consumer half of hugetlb swap support; the producer (swap-out via MADV_PAGEOUT) arrives in a later patch, so every path added here is provably unreachable until then - which is exactly why it lands first: with swap-out first, a fault on a swapped-out hugetlb page would hit the 'ret = 0' fall-through in hugetlb_fault() and livelock.
* hugetlb_anon_swapin(), mirroring do_swap_page(): get_swap_device() pins the device, swap_cache_get_folio() finds in-flight folios, and on a miss the folio is allocated from the hugetlb pool and added to the swap cache via the new generic swap_cache_add_folio() helper (hugetlb folios cannot come from the buddy allocator, so __swap_cache_alloc() cannot be used). Pool exhaustion retries briefly, then fails with SIGBUS, matching the reservation contract of the no-page path. A hwpoisoned swapcache folio kills the faulter with VM_FAULT_HWPOISON_LARGE instead of being delivered.
* Fork: copy_hugetlb_page_range() duplicates the slot references of a swapped-out page with the new swap_dup_entries_direct() helper (the swap cache folio cannot be used: it is freed back to the pool once writeback completes while its slots stay pinned by the entries) and clears the exclusive bit on both sides, like copy_nonpresent_pte().
* Zap: __unmap_hugepage_range() returns the slot references with swap_put_entries_direct(); its present-folio branch additionally drains a leftover swapcache copy (anon, unmapped, non-poisoned) via folio_free_swap(), since hugetlb folios never sit on an LRU and vmscan cannot reclaim them later.
* hugetlb_wp() refuses to reuse a swapcache folio in place unless folio_free_swap() proves no slot is referenced anymore (fork may have duplicated the entries); hugetlb_change_protection() passes swap entries through, only the uffd-wp bit may change.
* MM_SWAPENTS follows the mainline contract (+N at swapout and fork, -N at swapin and zap, N = pages per huge page) so VmSwap stays meaningful.
* swapoff: hugetlb_unuse_vma() walks one VMA's swap entries of the dying type and swaps them in through the same worker, wired into unuse_mm(); try_to_unuse() must be able to drain every kind of swapped-out page.
* pagemap: report hugetlb swap entries with PM_SWAP, soft-dirty and uffd-wp bits, like small-page swap entries.
--- fs/proc/task_mmu.c | 11 ++ include/linux/hugetlb.h | 2 + include/linux/swap.h | 1 + mm/hugetlb.c | 414 +++++++++++++++++++++++++++++++++++++++- mm/swap.h | 6 + mm/swap_state.c | 32 ++++ mm/swapfile.c | 66 +++++++ 7 files changed, 531 insertions(+), 1 deletion(-)
fs/proc/task_mmu.c +11 -0
@@ -2161,6 +2161,17 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,if (pm->show_pfn) frame =pte_pfn (pte) + ((addr & ~hmask) >>PAGE_SHIFT ); + }else if (softleaf_is_swap (softleaf_from_pte (pte))) { +softleaf_t entry =softleaf_from_pte (pte); + +if (pte_swp_soft_dirty (pte)) + flags |=PM_SOFT_DIRTY ; +if (pte_swp_uffd_any (pte)) + flags |=PM_UFFD_WP ; +if (pm->show_pfn) + frame =swp_type (entry) | + (swp_offset (entry) <<MAX_SWAPFILES_SHIFT ); + flags |=PM_SWAP ; }else if (pte_swp_uffd_any (pte)) { flags |=PM_UFFD_WP ; }
include/linux/hugetlb.h +2 -0
@@ -1377,4 +1377,6 @@ hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz) return huge_pte_offset(vma->vm_mm, addr, sz); } +int hugetlb_unuse_vma (struct vm_area_struct *vma,unsigned int type); +#endif /* _LINUX_HUGETLB_H */
include/linux/swap.h +1 -0
@@ -414,6 +414,7 @@ sector_t swap_folio_sector(struct folio *folio);* a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct (swp_entry_t entry); +int swap_dup_entries_direct (swp_entry_t entry,int nr);void swap_put_entries_direct (swp_entry_t entry,int nr);/*
mm/hugetlb.c +413 -1
480 lines not on this page · raw message
mm/swap.h +6 -0
@@ -314,6 +314,7 @@ bool swap_cache_has_folio(swp_entry_t entry);struct folio *swap_cache_get_folio (swp_entry_t entry);void *swap_cache_get_shadow (swp_entry_t entry);void swap_cache_del_folio (struct folio *folio); +int swap_cache_add_folio (struct folio *folio,swp_entry_t entry);struct folio *swap_cache_alloc_folio (swp_entry_t target_entry,gfp_t gfp_mask,unsigned long orders,struct vm_fault *vmf,struct mempolicy *mpol,pgoff_t ilx);
@@ -396,6 +397,11 @@ static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entryreturn false; } +static inline int swap_cache_add_folio (struct folio *folio,swp_entry_t entry) +{ +return -EINVAL ; +} +static inline void show_swap_cache_info (void ) { }
mm/swap_state.c +32 -0
@@ -258,6 +258,38 @@ void __swap_cache_add_folio(struct swap_cluster_info *ci,lruvec_stat_mod_folio (folio,NR_SWAPCACHE , nr_pages); } +/** +* swap_cache_add_folio - Add an externally allocated folio to the swap cache. +* @folio: The folio to add, must be locked and swapbacked. +* @entry: The target swap entry, will be rounded down to the folio size. +* +* Mirrors __swap_cache_alloc() for callers that allocate the folio +* themselves, e.g. hugetlb whose folios must come from the hugetlb pool. +* +* Context: Caller must ensure @entry is valid and stabilize the swap +* device, e.g. via get_swap_device(). +* Return: 0 on success, -ENOENT if the slot is not swapped out, -EEXIST +* if it is already cached, -EBUSY on a conflicting concurrent operation. +*/ +int swap_cache_add_folio (struct folio *folio,swp_entry_t entry) +{ +struct swap_cluster_info *ci; +unsigned long nr_pages =folio_nr_pages (folio); +int err; + +VM_WARN_ON_ONCE_FOLIO (!folio_test_locked (folio), folio); +VM_WARN_ON_ONCE (nr_pages >SWAPFILE_CLUSTER ); + + entry.val =round_down (entry.val, nr_pages); + ci =swap_cluster_lock (__swap_entry_to_info (entry),swp_offset (entry)); + err =__swap_cache_add_check (ci, entry, nr_pages,NULL ,NULL ); +if (!err) +__swap_cache_add_folio (ci, folio, entry); +swap_cluster_unlock (ci); + +return err; +} +static void __swap_cache_do_del_folio (struct swap_cluster_info *ci,struct folio *folio,swp_entry_t entry,void *shadow)
mm/swapfile.c +66 -0
78 lines not on this page · raw message
Signature · 1 line
2.53.0
RFC PATCH 3/6 mm/hugetlb: swap out anonymous hugetlb folios via MADV_PAGEOUT
In reply to a message not in the archiveAdd the producer side of hugetlb swap: private anonymous hugetlb folios (MAP_PRIVATE hugetlbfs/memfd mappings after COW) can now be swapped out on explicit userspace request via madvise(2) MADV_PAGEOUT / process_madvise(2). The kernel never swaps hugetlb pages on its
own: cold-page scoring and swap decisions belong to userspace.
The pieces:
- hugetlb_reclaim_pages() and hugetlb_reclaim_folio_list(): a deliberate mirror of reclaim_pages()/shrink_folio_list(), not a reuse. The folio source (hstate activelists vs LRUs), the free path (free_huge_folio() vs free_unref_page_list()), the putback and the swap-cache removal (swap cluster lock vs i_pages) all differ, and hugetlb carries no scan_control/memcg-stat/demotion/workingset state. All reusable leaf operations (rmap walk, slot allocation, writeout, referenced/pin checks) are shared; only the control flow is mirrored, so a future unification can mechanically fold the two together. The machinery lives in hugetlb.c because it is pool
management: folios come from folio_isolate_hugetlb() and go back via free_huge_folio() or the per-hstate active list, all under hugetlb_lock -- the same family as demote_pool_huge_page() and set_max_huge_pages().
- try_to_unmap_swap_hugetlb_one(): an rmap walker callback mirroring ttu_anon_swapbacked_folio() that replaces the huge PTE with a swap entry covering the whole folio (single walk, swp_pte_prepare(), mm_prepare_for_swap_entries(), MM_SWAPENTS accounting), plus the exported wrapper try_to_unmap_swap_hugetlb(), following the try_to_unmap_poisoned_hugetlb_one() precedent. Cost to generic rmap code: one added line in rmap.h, no changes to existing code.
- hugetlb_folio_swap_supported(): the size gate. Upper bound: swap space is allocated in ranges of at most SWAPFILE_CLUSTER (PMD size) pages, so gigantic folios cannot be swapped. Lower bound: while swap cached, the HPG_* per-folio flags live in the low bits of folio->swap.val and the entry is recovered by rounding it down to the folio size (see folio_swap_entry()), which requires nr_pages >= 2^__NR_HPAGEFLAGS. Relaxing the lower bound flag by flag breaks down at bit 5 (raw_hwp_unreliable, set on the memory-failure kmalloc-failure fallback path, independently of the folio size): extremely unlikely, but the consequence would be a shifted swap offset, i.e. silent data corruption, and correctness cannot rest on probability. A VM_WARN_ON_ONCE_FOLIO in the swap-cache add path backstops the gate. The gate scopes a new feature rather than removing anything: upstream supports no hugetlb swap at any size today; smaller sizes are left for a dedicated entry store follow-up.
- madvise: MADV_PAGEOUT on a private hugetlb VMA isolates the present hugepages and hands them to hugetlb_reclaim_pages(). Shared mappings (supported later in this series), userfaultfd-registered or locked VMAs, and hstates failing the size gate are rejected with EINVAL.
This patch is the producer; the consumer (fault/swapoff swap-in) landed in the previous patch. The order matters and must not be
flipped: without the swap-in side, hugetlb_fault() returns success without installing anything for non-present entries that are neither migration nor hwpoison entries, so if page-out came first, touching a swapped-out page would fault in an endless loop.
--- include/linux/hugetlb.h | 36 +++++ include/linux/rmap.h | 1 + mm/hugetlb.c | 317 ++++++++++++++++++++++++++++++++++++++++ mm/madvise.c | 76 ++++++++++ mm/rmap.c | 147 +++++++++++++++++++ mm/swap_state.c | 8 + 6 files changed, 585 insertions(+)
include/linux/hugetlb.h +36 -0
@@ -866,6 +866,30 @@ static inline struct hstate *folio_hstate(struct folio *folio)return size_to_hstate (folio_size (folio)); } +/* +* Whether a hugetlb folio can be swapped out, i.e. enter the swap cache. +* +* Lower bound: while the folio is swap cached, the HPG_* per-folio flags +* are preserved in the low bits of folio->swap.val and the entry is +* recovered by rounding it down to the folio size (see folio_swap_entry()). +* That only works when the folio is large enough for all flag bits to fit +* below the swap offset. +* +* Upper bound: swap space is allocated in ranges of at most +* SWAPFILE_CLUSTER (PMD size) pages, so gigantic or larger folios cannot +* be swapped. +*/ +static inline bool hugetlb_folio_swap_supported (const struct folio *folio) +{ +unsigned int order =folio_order (folio); + +if (folio_nr_pages (folio) < (1UL << __NR_HPAGEFLAGS)) +return false; +if (order_is_gigantic (order) || order >HPAGE_PMD_ORDER ) +return false; +return true; +} +static inline unsigned hstate_index_to_shift (unsigned index) {return hstates[index].order +PAGE_SHIFT ;
@@ -1195,6 +1219,11 @@ static inline bool hstate_is_gigantic(struct hstate *h)return false; } +static inline bool hugetlb_folio_swap_supported (const struct folio *folio) +{ +return false; +} +static inline unsigned int pages_per_huge_page (struct hstate *h) {return 1 ;
@@ -1377,6 +1406,13 @@ hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz) return huge_pte_offset(vma->vm_mm, addr, sz); } +/* +* Self-contained reclaim of isolated hugetlb folios. Hugetlb folios are not +* on the normal LRU, so they are swapped out here instead of through the +* generic reclaim_pages()/shrink_folio_list() path. +*/ +unsigned long hugetlb_reclaim_pages (struct list_head *folio_list); +int hugetlb_unuse_vma (struct vm_area_struct *vma,unsigned int type);#endif /* _LINUX_HUGETLB_H */
include/linux/rmap.h +1 -0
@@ -966,6 +966,7 @@ struct rmap_walk_control {bool (*invalid_vma)(struct vm_area_struct *vma,void *arg); }; +void try_to_unmap_swap_hugetlb (struct folio *folio);void rmap_walk (struct folio *folio,struct rmap_walk_control *rwc);void rmap_walk_locked (struct folio *folio,struct rmap_walk_control *rwc); struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
mm/hugetlb.c +317 -0
323 lines not on this page · raw message
mm/madvise.c +76 -0
87 lines not on this page · raw message
mm/rmap.c +147 -0
159 lines not on this page · raw message
mm/swap_state.c +8 -0
@@ -21,6 +21,7 @@#include <linux/migrate.h> #include <linux/vmalloc.h> #include <linux/huge_mm.h> +#include <linux/hugetlb.h> #include <linux/shmem_fs.h> #include <linux/sysctl.h> #include <linux/swap_ops.h>
@@ -217,6 +218,13 @@ static void __swap_cache_do_add_folio(struct swap_cluster_info *ci,VM_WARN_ON_ONCE_FOLIO (!folio_test_locked (folio), folio);VM_WARN_ON_ONCE_FOLIO (folio_test_swapcache (folio), folio);VM_WARN_ON_ONCE_FOLIO (!folio_test_swapbacked (folio), folio); +/* +* Hugetlb folios too small for all per-folio flag bits to fit below +* the swap offset must never reach the swap cache; they are kept +* out by the hugetlb reclaim path (see folio_swap_entry()). +*/ +VM_WARN_ON_ONCE_FOLIO (folio_test_hugetlb (folio) && + !hugetlb_folio_swap_supported (folio), folio); ci_end = ci_off + nr_pages; do {
Signature · 1 line
2.53.0
RFC PATCH 4/6 mm/memory-failure: handle swapcached hugetlb folios
In reply to a message not in the archiveAnonymous hugetlb folios can now sit in the swap cache (swap-out writeback window, cached copy after swap-in), a state memory failure never had to handle before. Two gaps:
1. try_to_unmap() routes every hugetlb folio to try_to_unmap_poisoned_hugetlb_one(), which requires TTU_HWPOISON. But unmap_poisoned_folio() clears TTU_HWPOISON for a dirty swapcache folio -- its mappings must be replaced with swap entries, exactly like for a 4K swapcache folio, so the swap accounting stays intact and the fault path gets to kill the owner. Routing such a folio to the poisoned handler fires its VM_WARN_ON_ONCE and would install hwpoison entries on top of live swap state. Route hugetlb folios by flag instead: TTU_HWPOISON set -> the poisoned handler; cleared -> try_to_unmap_swap_hugetlb_one().
2. me_huge_page() has no swapcache case: a clean poisoned hugetlb folio in the swap cache falls into the truncate path, which evicts it (the swap address space has no error_remove_folio). The folio is never returned to the hstate pool (a silent 2M pool leak) and the poison marker is lost, so a later fault would swap possibly corrupt data back in. Mirror me_swapcache_dirty(): keep the poisoned folio in the swap cache and return MF_DELAYED, so the swap-in path sees folio_test_hwpoison() and kills the accessor.
--- mm/memory-failure.c | 19 +++++++++++++++++++ mm/rmap.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 6 deletions(-)
mm/memory-failure.c +19 -0
@@ -1151,6 +1151,25 @@ static int me_huge_page(struct page_state *ps, struct page *p)struct address_space *mapping;bool extra_pins = false; +/* +* A hugetlb folio reaches the swap cache only via hugetlb +* swap-out. Keep a poisoned one there so that the swap-in +* fault path intercepts folio_test_hwpoison() and kills the +* accessor. The truncate path below would evict a clean one +* (the swap address space has no error_remove_folio), losing +* both the folio (it is never returned to the hstate pool) and +* the poison marker, so a later fault would silently swap +* possibly-corrupt data back in. Mirror me_swapcache_dirty(). +*/ +if (folio_test_swapcache (folio)) { +folio_clear_dirty (folio); +folio_unlock (folio); +/* The swap cache pin is intentionally retained. */ +if (has_extra_refcount (ps, p, true)) +return MF_FAILED ; +return MF_DELAYED ; + } + mapping =folio_mapping (folio);if (mapping) { res =truncate_error_folio (folio,page_to_pfn (p), mapping);
mm/rmap.c +26 -6
@@ -1991,8 +1991,9 @@ static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio,pte_t pteval;/* -* The try_to_unmap() is only passed a hugetlb folio in the case -* where the hugetlb folio is poisoned. +* try_to_unmap() routes a hugetlb folio here only for memory +* failure with TTU_HWPOISON set; a poisoned folio kept in the +* swap cache is routed to try_to_unmap_swap_hugetlb_one() instead. */ VM_WARN_ON_ONCE_FOLIO (!folio_test_hwpoison (folio), folio);VM_WARN_ON_ONCE (!(flags &TTU_HWPOISON ));
@@ -2199,8 +2200,10 @@ static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,* with a swap entry covering the whole folio; for a file-backed folio the * PTE is simply cleared, the swap anchor lives in the hugetlbfs page * cache instead (shmem-style). Called from the hugetlb reclaim path -* (hugetlb_reclaim_pages()) with the folio lock held; an anonymous folio -* is swapbacked with its swap slots allocated. +* (hugetlb_reclaim_pages()) and, via try_to_unmap(), from memory failure +* for a poisoned folio that is kept in the swap cache (see +* unmap_poisoned_folio()); the folio lock is held in all cases, and an +* anonymous folio is swapbacked with its swap slots allocated. * * Keep in sync with ttu_anon_swapbacked_folio(). */
@@ -2567,13 +2570,30 @@ static int folio_not_mapped(struct folio *folio)void try_to_unmap (struct folio *folio,enum ttu_flags flags) {struct rmap_walk_control rwc = { - .rmap_one =folio_test_hugetlb (folio) ? - try_to_unmap_poisoned_hugetlb_one : try_to_unmap_one, + .rmap_one = try_to_unmap_one, .arg = (void *)flags, .done = folio_not_mapped, .anon_lock = folio_lock_anon_vma_read, }; +/* +* try_to_unmap() is passed a hugetlb folio only by memory failure. +* With TTU_HWPOISON the mappings are replaced with hwpoison +* entries. Without it the folio is a poisoned folio that is kept +* in the swap cache (unmap_poisoned_folio() cleared TTU_HWPOISON), +* so the mappings are replaced with swap entries, exactly like for +* a 4K swapcache folio: the swap accounting stays intact and the +* fault path gets to kill the owner. +*/ +if (folio_test_hugetlb (folio)) { +if (flags &TTU_HWPOISON ) + rwc.rmap_one = try_to_unmap_poisoned_hugetlb_one; +else if (WARN_ON_ONCE (!folio_test_swapcache (folio))) +return ; +else + rwc.rmap_one = try_to_unmap_swap_hugetlb_one; + } +if (flags &TTU_RMAP_LOCKED )rmap_walk_locked (folio, &rwc);else
Signature · 1 line
2.53.0
RFC PATCH 5/6 mm/hugetlb: swap support for file-backed hugetlb folios
In reply to a message not in the archiveExtend hugetlb swap-out to shared file-backed mappings, using the shmem model: swap-out only clears the PTEs -- no swap entry is ever installed for a file folio -- and the anchor (a swap value entry, swp_to_radix_entry()) is left in the hugetlbfs page cache, holding a swap count reference on the slots. Without this, instantiated shared huge pages pin the pool absolutely until truncate/unlink: QEMU guests backed by MAP_SHARED hugetlbfs files (-mem-path) are the canonical victim. It also fixes an ABI asymmetry userspace can trip over:MADV_PAGEOUT succeeds on private hugetlb and every 4K/THP mapping including shmem, but used to fail with EINVAL on shared hugetlb.
The pieces, each mirroring its shmem counterpart:
- Swap-out: hugetlbfs_writeout() allocates the folio-sized slot range, links the inode on the swapoff traversal list, moves the anchor into the page cache in place of the folio (holding a dup'ed swap count), and writes out. The reclaim loop drops its anonymous-only gate and MADV_PAGEOUT drops the VM_MAYSHARE rejection.
- Refault swap-in: hugetlb_no_page() recognizes the anchor (the page cache lookup reports it as a hole) and swaps the folio back in via hugetlbfs_do_swapin(), reinstalling it in place of the anchor -- indistinguishable from a page-cache hit afterwards. This consumer must never lag the producer: the base fault path treats value entries as holes and would silently zero-fill over swapped data.
- read(2) swap-in: hugetlbfs_read_iter() resolves anchors through hugetlbfs_swapin_read() instead of zero-filling; swapin failures surface as -EIO/-EFAULT, never stale data.
- Lifecycle: truncate/hole-punch/evict free anchors and their slots via hugetlbfs_free_swap(), with UAF guards (cache entry removed before the anchor's swap count is dropped; a concurrent reclaim holding the folio is waited out). Inode eviction drains the swapoff traversal list.
- swapoff: try_to_unuse() calls hugetlbfs_unuse() right after shmem_unuse(); since file folios install no swap PTEs, the mm walk cannot reach their slots -- the per-inode swaplist traversal can.
Accounting stays symmetric with free_huge_folio() on every path (hugetlb_cgroup, memcg swap/hugetlb charge, global reserve, NR_HUGETLB); the vma-less read/swapoff swapin charges the current context, mirroring shmem swapoff.
--- fs/hugetlbfs/inode.c | 128 ++++++- include/linux/hugetlb.h | 20 ++ include/linux/pagemap.h | 7 + mm/hugetlb.c | 733 ++++++++++++++++++++++++++++++++++++++-- mm/internal.h | 19 +- mm/madvise.c | 14 +- mm/swapfile.c | 7 + 7 files changed, 882 insertions(+), 46 deletions(-)
fs/hugetlbfs/inode.c +123 -5
@@ -250,6 +250,42 @@ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)/* Find the folio */ folio =filemap_lock_hugetlb_folio (h, mapping, index); +if (IS_ERR (folio)) { +/* +* The lookup reports a swapped-out page (swap anchor +* value entry) as a hole; try to swap it back in +* before falling back to zero-fill. +*/ +if (PTR_ERR (folio) == -ENOENT ) { + folio =hugetlbfs_swapin_read (inode, index); +if (IS_ERR (folio)) { +int err =PTR_ERR (folio); + +/* Raced with a concurrent swapin. */ +if (err == -EAGAIN ) +continue ; +/* +* -ENOENT is a genuine hole: fall +* through and zero-fill. Anything +* else is a swapin failure that must +* surface as a read error, never +* stale/zeroed data. +*/ +if (err != -ENOENT ) { +/* A corrupt page or bad swap +* entry reads as -EIO. */ +if (err == -EHWPOISON || + err == -EINVAL ) + retval = -EIO ; +/* POSIX: only report if nothing +* was copied yet. */ +if (!retval) + retval = -EFAULT ; +break ; + } + } + } + }if (IS_ERR (folio)) {/* * We have a HOLE, zero out the user-buffer for the
@@ -494,8 +530,9 @@ hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start,/* * Called with hugetlb fault mutex held. +* Returns true if the folio was actually removed, false otherwise. */ -static void remove_inode_single_folio (struct hstate *h,struct inode *inode, +static bool remove_inode_single_folio (struct hstate *h,struct inode *inode,struct address_space *mapping,struct folio *folio,pgoff_t index,bool truncate_op) {
@@ -508,6 +545,20 @@ static void remove_inode_single_folio(struct hstate *h, struct inode *inode,* lock to guarantee no concurrent migration. */ folio_lock (folio); + +/* +* Re-check the mapping under the folio lock: a concurrent +* hugetlbfs_writeout() may have replaced this folio with a +* swap anchor in the page cache and cleared folio->mapping +* before we acquired the lock. In that case the folio is no +* longer ours to remove; the anchor will be handled when +* remove_inode_hugepages() iterates the page cache again. +*/ +if (unlikely (folio_mapping (folio) != mapping)) { +folio_unlock (folio); +return false; + } +if (unlikely (folio_mapped (folio)))hugetlb_unmap_file_folio (h, mapping, folio, index);
@@ -527,6 +578,7 @@ static void remove_inode_single_folio(struct hstate *h, struct inode *inode, }folio_unlock (folio); +return true; }/*
@@ -556,30 +608,72 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart,struct address_space *mapping = &inode->i_data;const pgoff_t end = lend >>PAGE_SHIFT ;struct folio_batch fbatch; +pgoff_t indices[FOLIO_BATCH_SIZE ];pgoff_t next, index;int i, freed =0 ;bool truncate_op = (lend ==LLONG_MAX );folio_batch_init (&fbatch); next = lstart >>PAGE_SHIFT ; -while (filemap_get_folios (mapping, &next, end -1 , &fbatch)) { +while (find_get_entries (mapping, &next, end -1 , &fbatch, indices)) {for (i =0 ; i <folio_batch_count (&fbatch); ++i) {struct folio *folio = fbatch.folios[i];u32 hash =0 ; - index = folio->index >>huge_page_order (h); + index = indices[i] >>huge_page_order (h); hash =hugetlb_fault_mutex_hash (mapping, index);mutex_lock (&hugetlb_fault_mutex_table[hash]); +if (xa_is_value (folio)) { +/* +* Swap anchor of a swapped-out file page. +* hugetlbfs page caches carry no other +* value entries. +*/ +if (!hugetlbfs_free_swap (mapping, indices[i], + folio, h)) { +/* Entry changed; rescan from here. */ + next = indices[i]; +mutex_unlock (&hugetlb_fault_mutex_table[hash]); +break ; + } + +/* +* Account anchors like folios: unreserve +* per page for hole punch, count and +* unreserve once at the end for truncate. +*/ +if (!truncate_op) { +if (unlikely (hugetlb_unreserve_pages (inode, + index, index +1 ,1 ))) +hugetlb_fix_reserve_counts (inode); + }else { + freed++; + } + +mutex_unlock (&hugetlb_fault_mutex_table[hash]); +continue ; + } +/* * Remove folio that was part of folio_batch. */ -remove_inode_single_folio (h, inode, mapping, folio, - index, truncate_op); +if (!remove_inode_single_folio (h, inode, mapping, folio, + index, truncate_op)) { +/* +* Raced with swapout: the folio was +* replaced by a swap anchor. Rescan +* from this index to find it. +*/ + next = indices[i]; +mutex_unlock (&hugetlb_fault_mutex_table[hash]); +break ; + } freed++;mutex_unlock (&hugetlb_fault_mutex_table[hash]); } +folio_batch_remove_exceptionals (&fbatch);folio_batch_release (&fbatch);cond_resched (); }
@@ -596,6 +690,7 @@ static void hugetlbfs_evict_inode(struct inode *inode)trace_hugetlbfs_evict_inode (inode);remove_inode_hugepages (inode,0 ,LLONG_MAX ); +hugetlbfs_evict_drain_swaplist (inode); resv_map =HUGETLBFS_I (inode)->resv_map;/* Only regular and link inodes have associated reserve maps */
@@ -782,6 +877,25 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,continue ; } +/* +* A swap anchor means the page is swapped out: it is +* already backed (on swap), so there is nothing to +* preallocate. filemap_get_folio() reports anchors as +* holes, so check explicitly. The fault mutex held here +* keeps the anchor stable. +*/ + { +void *entry =filemap_get_entry (mapping, + index <<huge_page_order (h)); + +if (entry) { +if (!xa_is_value (entry)) +folio_put (entry); +mutex_unlock (&hugetlb_fault_mutex_table[hash]); +continue ; + } + } +/* * Allocate folio without setting the avoid_reserve argument. * There certainly are no reserves associated with the
@@ -921,6 +1035,10 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,simple_inode_init_ts (inode); info->resv_map = resv_map; info->seals =F_SEAL_SEAL ; +/* Initialize swapoff support fields */ +INIT_LIST_HEAD (&info->swaplist); +atomic_set (&info->swapped,0 ); +atomic_set (&info->stop_eviction,0 );switch (mode &S_IFMT ) {default :init_special_inode (inode, mode, dev);
include/linux/hugetlb.h +20 -0
@@ -508,6 +508,9 @@ struct hugetlbfs_inode_info {struct inode vfs_inode;struct resv_map *resv_map;unsigned int seals; +struct list_head swaplist;/* Link to hugetlbfs_swaplist */ +atomic_t swapped;/* Count of swapped pages */ +atomic_t stop_eviction;/* Prevent eviction during swapoff */ };static inline struct hugetlbfs_inode_info *HUGETLBFS_I (struct inode *inode)
@@ -890,6 +893,23 @@ static inline bool hugetlb_folio_swap_supported(const struct folio *folio)return true; } +/* +* File-page swap anchor removal for truncate/hole-punch/evict, and the +* evict-time drain of the swapoff traversal list. Defined in mm/hugetlb.c, +* called from fs/hugetlbfs/inode.c. +*/ +long hugetlbfs_free_swap (struct address_space *mapping,pgoff_t index, +void *radswap,struct hstate *h); +void hugetlbfs_evict_drain_swaplist (struct inode *inode); + +/* +* Read-path swapin of a swapped-out hugetlbfs page (mm/hugetlb.c, +* called from fs/hugetlbfs/inode.c), and the swapoff traversal of all +* hugetlbfs swap anchors (called from try_to_unuse()). +*/ +struct folio *hugetlbfs_swapin_read (struct inode *inode,pgoff_t index); +int hugetlbfs_unuse (unsigned int type); +static inline unsigned hstate_index_to_shift (unsigned index) {return hstates[index].order +PAGE_SHIFT ;
include/linux/pagemap.h +7 -0
@@ -987,6 +987,13 @@ static inline bool folio_contains(const struct folio *folio, pgoff_t index)unsigned filemap_get_folios (struct address_space *mapping,pgoff_t *start,pgoff_t end,struct folio_batch *fbatch); +/* +* find_get_entries() also returns swap/shadow value entries (unlike +* filemap_get_folios()); hugetlbfs needs it to enumerate the swap +* anchors left in its page cache by hugetlb file-page swap-out. +*/ +unsigned find_get_entries (struct address_space *mapping,pgoff_t *start, +pgoff_t end,struct folio_batch *fbatch,pgoff_t *indices);unsigned filemap_get_folios_contig (struct address_space *mapping,pgoff_t *start,pgoff_t end,struct folio_batch *fbatch); unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
mm/hugetlb.c +705 -28
797 lines not on this page · raw message
mm/internal.h +16 -3
@@ -605,8 +605,6 @@ static inline void force_page_cache_readahead(struct address_space *mapping,unsigned find_lock_entries (struct address_space *mapping,pgoff_t *start,pgoff_t end,struct folio_batch *fbatch,pgoff_t *indices); -unsigned find_get_entries (struct address_space *mapping,pgoff_t *start, -pgoff_t end,struct folio_batch *fbatch,pgoff_t *indices);int truncate_inode_folio (struct address_space *mapping,struct folio *folio);bool truncate_inode_partial_folio (struct folio *folio,loff_t start,loff_t end);
@@ -1469,8 +1467,23 @@ static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,/* Only track the nodes of mappings with shadow entries */ void workingset_update_node (struct xa_node *node);extern struct list_lru shadow_nodes; + +#ifdef CONFIG_HUGETLBFS +static inline bool hugetlbfs_mapping (struct address_space *mapping) +{ +return mapping->host && + mapping->host->i_sb->s_magic ==HUGETLBFS_MAGIC ; +} +#else +static inline bool hugetlbfs_mapping (struct address_space *mapping) +{ +return false; +} +#endif /* CONFIG_HUGETLBFS */ +#define mapping_set_update (xas, mapping) do { \ - if (!dax_mapping(mapping) && !shmem_mapping(mapping)) { \ + if (!dax_mapping(mapping) && !shmem_mapping(mapping) && \ + !hugetlbfs_mapping(mapping)) { \ xas_set_update(xas, workingset_update_node); \ xas_set_lru(xas, &shadow_nodes); \ } \
mm/madvise.c +4 -10
@@ -636,9 +636,10 @@ static void madvise_pageout_page_range(struct mmu_gather *tlb, }/* -* Page out private anonymous hugetlb folios in the range: isolate each -* present hugepage, hand the batch to hugetlb_reclaim_pages() and let it -* unmap, write out and free the folios synchronously. +* Page out hugetlb folios in the range: isolate each present hugepage, +* hand the batch to hugetlb_reclaim_pages() and let it unmap, write out +* and free the folios synchronously. Both private (anonymous) folios +* and shared (hugetlbfs page-cache) folios are supported. */ static long madvise_pageout_hugetlb (struct madvise_behavior *madv_behavior) {
@@ -654,13 +655,6 @@ static long madvise_pageout_hugetlb(struct madvise_behavior *madv_behavior)return -EINVAL ;if (vma->vm_flags & (VM_LOCKED |VM_PFNMAP ))return -EINVAL ; -/* -* Swap PTEs are only installed for anonymous folios (MAP_PRIVATE -* after COW); shared hugetlbfs mappings gain swap-out support -* later in this series. -*/ -if (vma->vm_flags &VM_MAYSHARE ) -return -EINVAL ;/* A huge page must fit in a single swap cluster to be swappable. */ if (hstate_is_gigantic (h) ||huge_page_order (h) >HPAGE_PMD_ORDER )return -EINVAL ;
mm/swapfile.c +7 -0
@@ -2763,6 +2763,13 @@ static int try_to_unuse(unsigned int type)if (retval)return retval; +#ifdef CONFIG_HUGETLB_PAGE +/* Swap in any swapped-out hugetlbfs file pages (swap anchors). */ + retval =hugetlbfs_unuse (type); +if (retval) +return retval; +#endif + prev_mm = &init_mm;mmget (prev_mm);
Signature · 1 line
2.53.0
RFC PATCH 6/6 selftests/mm: add hugetlb_swap test, document hugetlb swap
In reply to a message not in the archiveAdd tools/testing/selftests/mm/hugetlb_swap.c covering the hugetlb swap feature end to end (37 assertions, 14 scenarios):
- anonymous pageout: PTE becomes a swap entry, the huge page returns to the pool, VmSwap grows by the huge page size
- fork sharing swap entries: duplicated slot references, MM_SWAPENTS charged to the child, COW on write, parent data unpolluted
- munmap/exit releasing swap slots; zap-time drain of the leftover swapcache copy after a read fault
- swapoff bringing swapped-out hugetlb pages back in
- pool exhaustion at swapin: SIGBUS after a bounded retry, matching the reservation contract
- hwpoison of a swapcache-resident folio: the folio stays in the swap cache and the accessor is killed with SIGBUS at swapin
- shared file-backed mappings: pageout clears the PTE without installing a swap entry, refault and read(2) both swap back in with data intact, repeated pageout/swapin cycles, truncate freeing the anchor and its slots (SIGBUS on subsequent access)
- reject paths: VM_LOCKED, userfaultfd-registered and gigantic VMAs fail MADV_PAGEOUT with EINVAL
- mprotect/mremap smoke over swap entries
- VmSwap accounting: +N at pageout, +N at fork, -N at swapin and zap (regression test for the unsigned-negation zap fix)
- soft-dirty and uffd-wp bit propagation through swapout/swapin
Verified on x86-64 (2M hugepages, dedicated swap device): 35 pass / 0 fail / 2 expected skips (mlock is a no-op on hugetlb; no free 1G gigantic pages).
Document the feature in hugetlbpage.rst: the userspace-driven PAGEOUT contract, the size and VMA restrictions, and the pool-exhaustion SIGBUS semantics a manager process must plan around.
--- Documentation/admin-guide/mm/hugetlbpage.rst | 29 +- tools/testing/selftests/mm/Makefile | 1 + tools/testing/selftests/mm/hugetlb_swap.c | 1118 ++++++++++++++++++ tools/testing/selftests/mm/run_vmtests.sh | 2 + 4 files changed, 1149 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/mm/hugetlb_swap.c
Documentation/admin-guide/mm/hugetlbpage.rst +28 -1
@@ -88,7 +88,8 @@ the user when the system is under memory pressure. Please try again later. Pages that are used as huge pages are reserved inside the kernel and cannot be used for other purposes. Huge pages cannot be swapped out under -memory pressure. +memory pressure; explicit userspace-driven swap-out is described in +Swapping_ below. Once a number of huge pages have been pre-allocated to the kernel huge page pool, a user with appropriate privilege can use either the mmap system call
@@ -469,6 +470,32 @@ errno set to EINVAL or exclude hugetlb pages that extend beyond the length if not hugepage aligned. For example, munmap(2) will fail if memory is backed by a hugetlb page and the length is smaller than the hugepage size. +Swapping +-------- + +Hugetlb mappings can be swapped out on explicit userspace request via +madvise(2) MADV_PAGEOUT / process_madvise(2). This covers both private +anonymous mappings (MAP_PRIVATE hugetlbfs files and MFD_HUGETLB memfds +after COW) and shared file-backed mappings; for shared mappings the +folios are written to swap while the PTEs are simply cleared, mirroring +the shmem model. The kernel never swaps hugetlb pages on its own: +cold-page scoring and swap decisions belong to userspace. + +Only huge pages that fit in a single swap cluster (i.e. PMD-sized, 2M on +x86-64 and arm64) are swappable; MADV_PAGEOUT on larger hstates, +userfaultfd-registered or locked VMAs fails with EINVAL. Swapped-out +folios are swapped back in on access (for shared mappings, read(2) +works as well). MADV_WILLNEED has no effect on hugetlb mappings: +hugetlbfs has no readahead support, so the advice is silently ignored +as it has always been. + +Swapping out releases the huge page back to the persistent pool, so a +later swap-in competes for pool memory. If the pool is exhausted at +fault time, the faulting task receives SIGBUS after a bounded retry, +matching the reservation-exhaustion contract. A manager process must +therefore create headroom (page out cold pages) before touching swapped +pages. + Examples ========
tools/testing/selftests/mm/Makefile +1 -0
@@ -66,6 +66,7 @@ TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline +TEST_GEN_FILES += hugetlb_swap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace
tools/testing/selftests/mm/hugetlb_swap.c new file +1118 -0
1,118 lines not on this page · raw message
tools/testing/selftests/mm/run_vmtests.sh +2 -0
@@ -266,6 +266,8 @@ CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv CATEGORY="hugetlb" run_test ./hugetlb_madv_vs_map +# needs a swap device and free 2M hugepages; skips cleanly without either +CATEGORY="hugetlb" run_test ./hugetlb_swap if test_selected "hugetlb"; then echo "NOTE: These hugetlb tests provide minimal coverage. Use" | tap_prefix
Signature · 1 line
2.53.0